Skip to contentSkip to conversation

RISC-V AIA irqchip and Svnapot support

PATCH Superseded Anup Patel 3 participants 18 messages Sep 18, 2023 – Nov 18, 2023

Subsystems (4)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Anup Patel

author
#0 lore raw

v2 cover RISC-V AIA irqchip and Svnapot support

The latest KVM in Linux-6.5 has support for:
1) Svnapot ISA extension support
2) AIA in-kernel irqchip support

This series adds corresponding changes in KVMTOOL to use the above mentioned features for Guest/VM.

These patches can also be found in the riscv_aia_v2 branch at:
https://github.com/avpatel/kvmtool.git

Changes since v1:
- Rebased on commit 9cb1b46cb765972326a46bdba867d441a842af56
- Updated PATCH1 to sync header with released Linux-6.5

Anup Patel (6):
Sync-up header with Linux-6.5 for KVM RISC-V
riscv: Add Svnapot extension support
riscv: Make irqchip support pluggable
riscv: Add IRQFD support for in-kernel AIA irqchip
riscv: Use AIA in-kernel irqchip whenever KVM RISC-V supports
riscv: Fix guest/init linkage for multilib toolchain

 Makefile                            |   3 +
 include/linux/kvm.h                 |   6 +-
 riscv/aia.c                         | 227 ++++++++++++++++++++++++++++
 riscv/fdt.c                         |  15 +-
 riscv/include/asm/kvm.h             |  81 ++++++++++
 riscv/include/kvm/fdt-arch.h        |   8 +-
 riscv/include/kvm/kvm-arch.h        |  38 ++++-
 riscv/include/kvm/kvm-config-arch.h |   3 +
 riscv/irq.c                         | 138 ++++++++++++++++-
 riscv/kvm.c                         |   2 +
 riscv/pci.c                         |  32 ++--
 riscv/plic.c                        |  61 ++++----
 12 files changed, 563 insertions(+), 51 deletions(-)
 create mode 100644 riscv/aia.c
Signature · 1 line
2.34.1
author
#1 lore raw

PATCH v2 1/6 Sync-up header with Linux-6.5 for KVM RISC-V

We sync-up Linux headers to get latest KVM RISC-V headers having V, Svnapot, and AIA support.

---
 include/linux/kvm.h     |  6 ++-
 riscv/include/asm/kvm.h | 81 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 86 insertions(+), 1 deletion(-)
include/linux/kvm.h +5 -1
index 737318b..f089ab2 100644
@@ -1190,6 +1190,8 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP 225
 #define KVM_CAP_PMU_EVENT_MASKED_EVENTS 226
 #define KVM_CAP_COUNTER_OFFSET 227
+#define KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE 228
+#define KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES 229
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1442,6 +1444,8 @@ enum kvm_device_type {
 #define KVM_DEV_TYPE_XIVE		KVM_DEV_TYPE_XIVE
 	KVM_DEV_TYPE_ARM_PV_TIME,
 #define KVM_DEV_TYPE_ARM_PV_TIME	KVM_DEV_TYPE_ARM_PV_TIME
+	KVM_DEV_TYPE_RISCV_AIA,
+#define KVM_DEV_TYPE_RISCV_AIA		KVM_DEV_TYPE_RISCV_AIA
 	KVM_DEV_TYPE_MAX,
 };
 
@@ -1613,7 +1617,7 @@ struct kvm_s390_ucas_mapping {
 #define KVM_GET_DEBUGREGS         _IOR(KVMIO,  0xa1, struct kvm_debugregs)
 #define KVM_SET_DEBUGREGS         _IOW(KVMIO,  0xa2, struct kvm_debugregs)
 /*
- * vcpu version available with KVM_ENABLE_CAP
+ * vcpu version available with KVM_CAP_ENABLE_CAP
  * vm version available with KVM_CAP_ENABLE_CAP_VM
  */
 #define KVM_ENABLE_CAP            _IOW(KVMIO,  0xa3, struct kvm_enable_cap)
riscv/include/asm/kvm.h +81 -0

99 lines not on this page · raw message

Signature · 1 line
2.34.1
author
#2 lore raw

PATCH v2 2/6 riscv: Add Svnapot extension support

In reply to Anup Patel

When the Svnapot extension is available expose it to the guest via device tree so that guest can use it.

---
 riscv/fdt.c                         | 1 +
 riscv/include/kvm/kvm-config-arch.h | 3 +++
 2 files changed, 4 insertions(+)
riscv/fdt.c +1 -0
index df71ed4..2724c6e 100644
@@ -19,6 +19,7 @@ struct isa_ext_info isa_info_arr[] = {
 	{"ssaia", KVM_RISCV_ISA_EXT_SSAIA},
 	{"sstc", KVM_RISCV_ISA_EXT_SSTC},
 	{"svinval", KVM_RISCV_ISA_EXT_SVINVAL},
+	{"svnapot", KVM_RISCV_ISA_EXT_SVNAPOT},
 	{"svpbmt", KVM_RISCV_ISA_EXT_SVPBMT},
 	{"zbb", KVM_RISCV_ISA_EXT_ZBB},
 	{"zicbom", KVM_RISCV_ISA_EXT_ZICBOM},
riscv/include/kvm/kvm-config-arch.h +3 -0
index b0a7e25..863baea 100644
@@ -34,6 +34,9 @@ struct kvm_config_arch {
 	OPT_BOOLEAN('\0', "disable-svinval",				\
 		    &(cfg)->ext_disabled[KVM_RISCV_ISA_EXT_SVINVAL],	\
 		    "Disable Svinval Extension"),			\
+	OPT_BOOLEAN('\0', "disable-svnapot",				\
+		    &(cfg)->ext_disabled[KVM_RISCV_ISA_EXT_SVNAPOT],	\
+		    "Disable Svnapot Extension"),			\
 	OPT_BOOLEAN('\0', "disable-svpbmt",				\
 		    &(cfg)->ext_disabled[KVM_RISCV_ISA_EXT_SVPBMT],	\
 		    "Disable Svpbmt Extension"),			\
Signature · 1 line
2.34.1
Reviewed-by on v2 2/6
#3 lore raw

Re: [kvmtool PATCH v2 2/6] riscv: Add Svnapot extension support

Quoted 38 lines · Anup Patel

When the Svnapot extension is available expose it to the guest via device tree so that guest can use it.

---
 riscv/fdt.c                         | 1 +
 riscv/include/kvm/kvm-config-arch.h | 3 +++
 2 files changed, 4 insertions(+)
riscv/fdt.c +1 -0
index df71ed4..2724c6e 100644
@@ -19,6 +19,7 @@ struct isa_ext_info isa_info_arr[] = {
 	{"ssaia", KVM_RISCV_ISA_EXT_SSAIA},
 	{"sstc", KVM_RISCV_ISA_EXT_SSTC},
 	{"svinval", KVM_RISCV_ISA_EXT_SVINVAL},
+	{"svnapot", KVM_RISCV_ISA_EXT_SVNAPOT},
 	{"svpbmt", KVM_RISCV_ISA_EXT_SVPBMT},
 	{"zbb", KVM_RISCV_ISA_EXT_ZBB},
 	{"zicbom", KVM_RISCV_ISA_EXT_ZICBOM},
riscv/include/kvm/kvm-config-arch.h +3 -0
index b0a7e25..863baea 100644
@@ -34,6 +34,9 @@ struct kvm_config_arch {
 	OPT_BOOLEAN('\0', "disable-svinval",				\
 		    &(cfg)->ext_disabled[KVM_RISCV_ISA_EXT_SVINVAL],	\
 		    "Disable Svinval Extension"),			\
+	OPT_BOOLEAN('\0', "disable-svnapot",				\
+		    &(cfg)->ext_disabled[KVM_RISCV_ISA_EXT_SVNAPOT],	\
+		    "Disable Svnapot Extension"),			\
 	OPT_BOOLEAN('\0', "disable-svpbmt",				\
 		    &(cfg)->ext_disabled[KVM_RISCV_ISA_EXT_SVPBMT],	\
 		    "Disable Svpbmt Extension"),			\
Signature · 1 line
2.34.1
author
#4 lore raw

PATCH v2 3/6 riscv: Make irqchip support pluggable

In reply to Anup Patel

We will be having different types of irqchip:
1) PLIC emulated by user-space
2) AIA APLIC and IMSIC provided by in-kernel KVM module

To support above, we de-couple PLIC specific code from generic RISC-V code (such as FDT generation) so that we can easily add other types of irqchip.

---
 riscv/fdt.c                  | 14 ++++++--
 riscv/include/kvm/kvm-arch.h | 25 ++++++++++++---
 riscv/irq.c                  | 62 ++++++++++++++++++++++++++++++++++--
 riscv/kvm.c                  |  2 ++
 riscv/pci.c                  | 32 +++++++++++++------
 riscv/plic.c                 | 61 +++++++++++++++++------------------
 6 files changed, 147 insertions(+), 49 deletions(-)
riscv/fdt.c +11 -3
index 2724c6e..9af71b5 100644
@@ -209,19 +209,26 @@ static int setup_fdt(struct kvm *kvm)
 	/* CPUs */
 	generate_cpu_nodes(fdt, kvm);
 
+	/* IRQCHIP */
+	if (!riscv_irqchip_generate_fdt_node)
+		die("No way to generate IRQCHIP FDT node\n");
+	riscv_irqchip_generate_fdt_node(fdt, kvm);
+
 	/* Simple Bus */
 	_FDT(fdt_begin_node(fdt, "smb"));
 	_FDT(fdt_property_string(fdt, "compatible", "simple-bus"));
 	_FDT(fdt_property_cell(fdt, "#address-cells", 0x2));
 	_FDT(fdt_property_cell(fdt, "#size-cells", 0x2));
-	_FDT(fdt_property_cell(fdt, "interrupt-parent", PHANDLE_PLIC));
+	_FDT(fdt_property_cell(fdt, "interrupt-parent",
+			       riscv_irqchip_phandle));
 	_FDT(fdt_property(fdt, "ranges", NULL, 0));
 
 	/* Virtio MMIO devices */
 	dev_hdr = device__first_dev(DEVICE_BUS_MMIO);
 	while (dev_hdr) {
 		generate_mmio_fdt_nodes = dev_hdr->data;
-		generate_mmio_fdt_nodes(fdt, dev_hdr, plic__generate_irq_prop);
+		generate_mmio_fdt_nodes(fdt, dev_hdr,
+					riscv__generate_irq_prop);
 		dev_hdr = device__next_dev(dev_hdr);
 	}
 
@@ -229,7 +236,8 @@ static int setup_fdt(struct kvm *kvm)
 	dev_hdr = device__first_dev(DEVICE_BUS_IOPORT);
 	while (dev_hdr) {
 		generate_mmio_fdt_nodes = dev_hdr->data;
-		generate_mmio_fdt_nodes(fdt, dev_hdr, plic__generate_irq_prop);
+		generate_mmio_fdt_nodes(fdt, dev_hdr,
+					riscv__generate_irq_prop);
 		dev_hdr = device__next_dev(dev_hdr);
 	}
 
riscv/include/kvm/kvm-arch.h +21 -4
index 660355b..cd37fc6 100644
@@ -10,8 +10,8 @@
 
 #define RISCV_IOPORT		0x00000000ULL
 #define RISCV_IOPORT_SIZE	SZ_64K
-#define RISCV_PLIC		0x0c000000ULL
-#define RISCV_PLIC_SIZE		SZ_64M
+#define RISCV_IRQCHIP		0x08000000ULL
+#define RISCV_IRQCHIP_SIZE		SZ_128M
 #define RISCV_MMIO		0x10000000ULL
 #define RISCV_MMIO_SIZE		SZ_512M
 #define RISCV_PCI		0x30000000ULL
@@ -84,10 +84,27 @@ static inline bool riscv_addr_in_ioport_region(u64 phys_addr)
 
 enum irq_type;
 
-void plic__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type);
+enum irqchip_type {
+	IRQCHIP_UNKNOWN = 0,
+	IRQCHIP_PLIC,
+	IRQCHIP_AIA
+};
+
+extern enum irqchip_type riscv_irqchip;
+extern bool riscv_irqchip_inkernel;
+extern void (*riscv_irqchip_trigger)(struct kvm *kvm, int irq,
+				     int level, bool edge);
+extern void (*riscv_irqchip_generate_fdt_node)(void *fdt, struct kvm *kvm);
+extern u32 riscv_irqchip_phandle;
+extern u32 riscv_irqchip_msi_phandle;
+extern bool riscv_irqchip_line_sensing;
 
-void plic__irq_trig(struct kvm *kvm, int irq, int level, bool edge);
+void plic__create(struct kvm *kvm);
 
 void pci__generate_fdt_nodes(void *fdt);
 
+void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type);
+
+void riscv__irqchip_create(struct kvm *kvm);
+
 #endif /* KVM__KVM_ARCH_H */
riscv/irq.c +60 -2

73 lines not on this page · raw message

riscv/kvm.c +2 -0
index 8daad94..1d49479 100644
@@ -96,6 +96,8 @@ void kvm__arch_init(struct kvm *kvm)
 
 	madvise(kvm->arch.ram_alloc_start, kvm->arch.ram_alloc_size,
 		MADV_HUGEPAGE);
+
+	riscv__irqchip_create(kvm);
 }
 
 #define FDT_ALIGN	SZ_4M
riscv/pci.c +23 -9
index 604fd20..61dee06 100644
@@ -7,20 +7,21 @@
 
 /*
  * An entry in the interrupt-map table looks like:
- * <pci unit address> <pci interrupt pin> <plic phandle> <plic interrupt>
+ * <pci unit address> <pci interrupt pin> <irqchip phandle> <irqchip line>
  */
 
 struct of_interrupt_map_entry {
 	struct of_pci_irq_mask		pci_irq_mask;
-	u32				plic_phandle;
-	u32				plic_irq;
+	u32				irqchip_phandle;
+	u32				irqchip_line;
+	u32				irqchip_sense;
 } __attribute__((packed));
 
 void pci__generate_fdt_nodes(void *fdt)
 {
 	struct device_header *dev_hdr;
 	struct of_interrupt_map_entry irq_map[OF_PCI_IRQ_MAP_MAX];
-	unsigned nentries = 0;
+	unsigned nentries = 0, nsize;
 	/* Bus range */
 	u32 bus_range[] = { cpu_to_fdt32(0), cpu_to_fdt32(1), };
 	/* Configuration Space */
@@ -48,6 +49,11 @@ void pci__generate_fdt_nodes(void *fdt)
 		},
 	};
 
+	/* Find size of each interrupt map entery */
+	nsize = sizeof(struct of_interrupt_map_entry);
+	if (!riscv_irqchip_line_sensing)
+		nsize -= sizeof(u32);
+
 	/* Boilerplate PCI properties */
 	_FDT(fdt_begin_node(fdt, "pci"));
 	_FDT(fdt_property_string(fdt, "device_type", "pci"));
@@ -64,12 +70,13 @@ void pci__generate_fdt_nodes(void *fdt)
 	/* Generate the interrupt map ... */
 	dev_hdr = device__first_dev(DEVICE_BUS_PCI);
 	while (dev_hdr && nentries < ARRAY_SIZE(irq_map)) {
-		struct of_interrupt_map_entry *entry = &irq_map[nentries];
+		struct of_interrupt_map_entry *entry;
 		struct pci_device_header *pci_hdr = dev_hdr->data;
 		u8 dev_num = dev_hdr->dev_num;
 		u8 pin = pci_hdr->irq_pin;
 		u8 irq = pci_hdr->irq_line;
 
+		entry = ((void *)irq_map) + (nsize * nentries);
 		*entry = (struct of_interrupt_map_entry) {
 			.pci_irq_mask = {
 				.pci_addr = {
@@ -79,16 +86,18 @@ void pci__generate_fdt_nodes(void *fdt)
 				},
 				.pci_pin	= cpu_to_fdt32(pin),
 			},
-			.plic_phandle	= cpu_to_fdt32(PHANDLE_PLIC),
-			.plic_irq	= cpu_to_fdt32(irq),
+			.irqchip_phandle	= cpu_to_fdt32(riscv_irqchip_phandle),
+			.irqchip_line		= cpu_to_fdt32(irq),
 		};
 
+		if (riscv_irqchip_line_sensing)
+			entry->irqchip_sense = cpu_to_fdt32(IRQ_TYPE_LEVEL_HIGH);
+
 		nentries++;
 		dev_hdr = device__next_dev(dev_hdr);
 	}
 
-	_FDT(fdt_property(fdt, "interrupt-map", irq_map,
-			  sizeof(struct of_interrupt_map_entry) * nentries));
+	_FDT(fdt_property(fdt, "interrupt-map", irq_map, nsize * nentries));
 
 	/* ... and the corresponding mask. */
 	if (nentries) {
@@ -105,5 +114,10 @@ void pci__generate_fdt_nodes(void *fdt)
 				  sizeof(irq_mask)));
 	}
 
+	/* Set MSI parent if available */
+	if (riscv_irqchip_msi_phandle != PHANDLE_RESERVED)
+		_FDT(fdt_property_cell(fdt, "msi-parent",
+				       riscv_irqchip_msi_phandle));
+
 	_FDT(fdt_end_node(fdt));
 }
riscv/plic.c +30 -31

122 lines not on this page · raw message

Signature · 1 line
2.34.1
Reviewed-by on v2 3/6
#5 lore raw

Re: [kvmtool PATCH v2 3/6] riscv: Make irqchip support pluggable

Quoted 52 lines · Anup Patel

We will be having different types of irqchip:
1) PLIC emulated by user-space
2) AIA APLIC and IMSIC provided by in-kernel KVM module

To support above, we de-couple PLIC specific code from generic RISC-V code (such as FDT generation) so that we can easily add other types of irqchip.

---
 riscv/fdt.c                  | 14 ++++++--
 riscv/include/kvm/kvm-arch.h | 25 ++++++++++++---
 riscv/irq.c                  | 62 ++++++++++++++++++++++++++++++++++--
 riscv/kvm.c                  |  2 ++
 riscv/pci.c                  | 32 +++++++++++++------
 riscv/plic.c                 | 61 +++++++++++++++++------------------
 6 files changed, 147 insertions(+), 49 deletions(-)
riscv/fdt.c +9 -2
index 2724c6e..9af71b5 100644
@@ -209,19 +209,26 @@ static int setup_fdt(struct kvm *kvm)
 	/* CPUs */
 	generate_cpu_nodes(fdt, kvm);
 
+	/* IRQCHIP */
+	if (!riscv_irqchip_generate_fdt_node)
+		die("No way to generate IRQCHIP FDT node\n");
+	riscv_irqchip_generate_fdt_node(fdt, kvm);
+
 	/* Simple Bus */
 	_FDT(fdt_begin_node(fdt, "smb"));
 	_FDT(fdt_property_string(fdt, "compatible", "simple-bus"));
 	_FDT(fdt_property_cell(fdt, "#address-cells", 0x2));
 	_FDT(fdt_property_cell(fdt, "#size-cells", 0x2));
-	_FDT(fdt_property_cell(fdt, "interrupt-parent", PHANDLE_PLIC));
+	_FDT(fdt_property_cell(fdt, "interrupt-parent",
+			       riscv_irqchip_phandle));
 	_FDT(fdt_property(fdt, "ranges", NULL, 0));
 
 	/* Virtio MMIO devices */
 	dev_hdr = device__first_dev(DEVICE_BUS_MMIO);
 	while (dev_hdr) {
 		generate_mmio_fdt_nodes = dev_hdr->data;
-		generate_mmio_fdt_nodes(fdt, dev_hdr, plic__generate_irq_prop);
+		generate_mmio_fdt_nodes(fdt, dev_hdr,
+					riscv__generate_irq_prop);
 		dev_hdr = device__next_dev(dev_hdr);
 	}
 

riscv/fdt.c static int setup_fdt(struct kvm *kvm) · line 242

Quoted 8 lines
riscv/fdt.c +2 -1
@@ -229,7 +236,8 @@ static int setup_fdt(struct kvm *kvm)
 	dev_hdr = device__first_dev(DEVICE_BUS_IOPORT);
 	while (dev_hdr) {
 		generate_mmio_fdt_nodes = dev_hdr->data;
-		generate_mmio_fdt_nodes(fdt, dev_hdr, plic__generate_irq_prop);
+		generate_mmio_fdt_nodes(fdt, dev_hdr,
+					riscv__generate_irq_prop);
 		dev_hdr = device__next_dev(dev_hdr);
 	}

nit: I'd let the above lines run long, but I know you prefer wrapping

     at 80.

riscv/include/kvm/kvm-arch.h line 14

Quoted 13 lines
riscv/include/kvm/kvm-arch.h +2 -2
index 660355b..cd37fc6 100644
@@ -10,8 +10,8 @@
 
 #define RISCV_IOPORT		0x00000000ULL
 #define RISCV_IOPORT_SIZE	SZ_64K
-#define RISCV_PLIC		0x0c000000ULL
-#define RISCV_PLIC_SIZE		SZ_64M
+#define RISCV_IRQCHIP		0x08000000ULL
+#define RISCV_IRQCHIP_SIZE		SZ_128M

I checked the applied patch and the SZ_128M is over indented.

Quoted 42 lines
riscv/include/kvm/kvm-arch.h +19 -2
 #define RISCV_MMIO		0x10000000ULL
 #define RISCV_MMIO_SIZE		SZ_512M
 #define RISCV_PCI		0x30000000ULL
@@ -84,10 +84,27 @@ static inline bool riscv_addr_in_ioport_region(u64 phys_addr)
 
 enum irq_type;
 
-void plic__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type);
+enum irqchip_type {
+	IRQCHIP_UNKNOWN = 0,
+	IRQCHIP_PLIC,
+	IRQCHIP_AIA
+};
+
+extern enum irqchip_type riscv_irqchip;
+extern bool riscv_irqchip_inkernel;
+extern void (*riscv_irqchip_trigger)(struct kvm *kvm, int irq,
+				     int level, bool edge);
+extern void (*riscv_irqchip_generate_fdt_node)(void *fdt, struct kvm *kvm);
+extern u32 riscv_irqchip_phandle;
+extern u32 riscv_irqchip_msi_phandle;
+extern bool riscv_irqchip_line_sensing;
 
-void plic__irq_trig(struct kvm *kvm, int irq, int level, bool edge);
+void plic__create(struct kvm *kvm);
 
 void pci__generate_fdt_nodes(void *fdt);
 
+void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type);
+
+void riscv__irqchip_create(struct kvm *kvm);
+
 #endif /* KVM__KVM_ARCH_H */
riscv/irq.c +1 -0
index 78a582d..b608a2f 100644
@@ -1,13 +1,71 @@
 #include "kvm/kvm.h"
 #include "kvm/kvm-cpu.h"
 #include "kvm/irq.h"
+#include "kvm/fdt.h"

riscv/irq.c line 14

Quoted 10 lines
riscv/irq.c +10 -0
+#include "kvm/virtio.h"
+
+enum irqchip_type riscv_irqchip = IRQCHIP_UNKNOWN;
+bool riscv_irqchip_inkernel = false;
+void (*riscv_irqchip_trigger)(struct kvm *kvm, int irq, int level, bool edge)
+				= NULL;
+void (*riscv_irqchip_generate_fdt_node)(void *fdt, struct kvm *kvm) = NULL;
+u32 riscv_irqchip_phandle = PHANDLE_RESERVED;
+u32 riscv_irqchip_msi_phandle = PHANDLE_RESERVED;
+bool riscv_irqchip_line_sensing = false;

nit: no need to init BSS symbols with zeros.

Quoted 116 lines
riscv/irq.c +49 -2
 void kvm__irq_line(struct kvm *kvm, int irq, int level)
 {
-	plic__irq_trig(kvm, irq, level, false);
+	struct kvm_irq_level irq_level;
+
+	if (riscv_irqchip_inkernel) {
+		irq_level.irq = irq;
+		irq_level.level = !!level;
+		if (ioctl(kvm->vm_fd, KVM_IRQ_LINE, &irq_level) < 0)
+			pr_warning("%s: Could not KVM_IRQ_LINE for irq %d\n",
+				   __func__, irq);
+	} else {
+		if (riscv_irqchip_trigger)
+			riscv_irqchip_trigger(kvm, irq, level, false);
+		else
+			pr_warning("%s: Can't change level for irq %d\n",
+				   __func__, irq);
+	}
 }
 
 void kvm__irq_trigger(struct kvm *kvm, int irq)
 {
-	plic__irq_trig(kvm, irq, 1, true);
+	if (riscv_irqchip_inkernel) {
+		kvm__irq_line(kvm, irq, VIRTIO_IRQ_HIGH);
+		kvm__irq_line(kvm, irq, VIRTIO_IRQ_LOW);
+	} else {
+		if (riscv_irqchip_trigger)
+			riscv_irqchip_trigger(kvm, irq, 1, true);
+		else
+			pr_warning("%s: Can't trigger irq %d\n",
+				   __func__, irq);
+	}
+}
+
+void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type)
+{
+	u32 prop[2], size;
+
+	prop[0] = cpu_to_fdt32(irq);
+	size = sizeof(u32);
+	if (riscv_irqchip_line_sensing) {
+		prop[1] = cpu_to_fdt32(irq_type);
+		size += sizeof(u32);
+	}
+
+	_FDT(fdt_property(fdt, "interrupts", prop, size));
+}
+
+void riscv__irqchip_create(struct kvm *kvm)
+{
+	/* Try PLIC irqchip */
+	plic__create(kvm);
+
+	/* Fail if irqchip unknown */
+	if (riscv_irqchip == IRQCHIP_UNKNOWN)
+		die("No IRQCHIP found\n");
 }
riscv/kvm.c +2 -0
index 8daad94..1d49479 100644
@@ -96,6 +96,8 @@ void kvm__arch_init(struct kvm *kvm)
 
 	madvise(kvm->arch.ram_alloc_start, kvm->arch.ram_alloc_size,
 		MADV_HUGEPAGE);
+
+	riscv__irqchip_create(kvm);
 }
 
 #define FDT_ALIGN	SZ_4M
riscv/pci.c +10 -4
index 604fd20..61dee06 100644
@@ -7,20 +7,21 @@
 
 /*
  * An entry in the interrupt-map table looks like:
- * <pci unit address> <pci interrupt pin> <plic phandle> <plic interrupt>
+ * <pci unit address> <pci interrupt pin> <irqchip phandle> <irqchip line>
  */
 
 struct of_interrupt_map_entry {
 	struct of_pci_irq_mask		pci_irq_mask;
-	u32				plic_phandle;
-	u32				plic_irq;
+	u32				irqchip_phandle;
+	u32				irqchip_line;
+	u32				irqchip_sense;
 } __attribute__((packed));
 
 void pci__generate_fdt_nodes(void *fdt)
 {
 	struct device_header *dev_hdr;
 	struct of_interrupt_map_entry irq_map[OF_PCI_IRQ_MAP_MAX];
-	unsigned nentries = 0;
+	unsigned nentries = 0, nsize;
 	/* Bus range */
 	u32 bus_range[] = { cpu_to_fdt32(0), cpu_to_fdt32(1), };
 	/* Configuration Space */
@@ -48,6 +49,11 @@ void pci__generate_fdt_nodes(void *fdt)
 		},
 	};
 
+	/* Find size of each interrupt map entery */
+	nsize = sizeof(struct of_interrupt_map_entry);
+	if (!riscv_irqchip_line_sensing)
+		nsize -= sizeof(u32);
+
 	/* Boilerplate PCI properties */
 	_FDT(fdt_begin_node(fdt, "pci"));
 	_FDT(fdt_property_string(fdt, "device_type", "pci"));
@@ -64,12 +70,13 @@ void pci__generate_fdt_nodes(void *fdt)
 	/* Generate the interrupt map ... */

riscv/pci.c void pci__generate_fdt_nodes(void *fdt) · line 79

Quoted 10 lines
riscv/pci.c +2 -1
 	dev_hdr = device__first_dev(DEVICE_BUS_PCI);
 	while (dev_hdr && nentries < ARRAY_SIZE(irq_map)) {
-		struct of_interrupt_map_entry *entry = &irq_map[nentries];
+		struct of_interrupt_map_entry *entry;
 		struct pci_device_header *pci_hdr = dev_hdr->data;
 		u8 dev_num = dev_hdr->dev_num;
 		u8 pin = pci_hdr->irq_pin;
 		u8 irq = pci_hdr->irq_line;
 
+		entry = ((void *)irq_map) + (nsize * nentries);

nit: The outer () of both terms are superfluous.

Quoted 128 lines
riscv/pci.c +11 -4
 		*entry = (struct of_interrupt_map_entry) {
 			.pci_irq_mask = {
 				.pci_addr = {
@@ -79,16 +86,18 @@ void pci__generate_fdt_nodes(void *fdt)
 				},
 				.pci_pin	= cpu_to_fdt32(pin),
 			},
-			.plic_phandle	= cpu_to_fdt32(PHANDLE_PLIC),
-			.plic_irq	= cpu_to_fdt32(irq),
+			.irqchip_phandle	= cpu_to_fdt32(riscv_irqchip_phandle),
+			.irqchip_line		= cpu_to_fdt32(irq),
 		};
 
+		if (riscv_irqchip_line_sensing)
+			entry->irqchip_sense = cpu_to_fdt32(IRQ_TYPE_LEVEL_HIGH);
+
 		nentries++;
 		dev_hdr = device__next_dev(dev_hdr);
 	}
 
-	_FDT(fdt_property(fdt, "interrupt-map", irq_map,
-			  sizeof(struct of_interrupt_map_entry) * nentries));
+	_FDT(fdt_property(fdt, "interrupt-map", irq_map, nsize * nentries));
 
 	/* ... and the corresponding mask. */
 	if (nentries) {
@@ -105,5 +114,10 @@ void pci__generate_fdt_nodes(void *fdt)
 				  sizeof(irq_mask)));
 	}
 
+	/* Set MSI parent if available */
+	if (riscv_irqchip_msi_phandle != PHANDLE_RESERVED)
+		_FDT(fdt_property_cell(fdt, "msi-parent",
+				       riscv_irqchip_msi_phandle));
+
 	_FDT(fdt_end_node(fdt));
 }
riscv/plic.c +11 -25
index 6242286..ab7c574 100644
@@ -118,7 +118,6 @@ struct plic_context {
 struct plic_state {
 	bool ready;
 	struct kvm *kvm;
-	struct device_header dev_hdr;
 
 	/* Static Configuration */
 	u32 num_irq;
@@ -204,7 +203,7 @@ static u32 __plic_context_irq_claim(struct plic_state *s,
 	return best_irq;
 }
 
-void plic__irq_trig(struct kvm *kvm, int irq, int level, bool edge)
+static void plic__irq_trig(struct kvm *kvm, int irq, int level, bool edge)
 {
 	bool irq_marked = false;
 	u8 i, irq_prio, irq_word;
@@ -425,7 +424,7 @@ static void plic__mmio_callback(struct kvm_cpu *vcpu,
 		die("plic: invalid len=%d", len);
 
 	addr &= ~0x3;
-	addr -= RISCV_PLIC;
+	addr -= RISCV_IRQCHIP;
 
 	if (is_write) {
 		if (PRIORITY_BASE <= addr && addr < ENABLE_BASE) {
@@ -464,34 +463,23 @@ static void plic__mmio_callback(struct kvm_cpu *vcpu,
 	}
 }
 
-void plic__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type)
-{
-	u32 irq_prop[] = {
-		cpu_to_fdt32(irq)
-	};
-
-	_FDT(fdt_property(fdt, "interrupts", irq_prop, sizeof(irq_prop)));
-}
-
-static void plic__generate_fdt_node(void *fdt,
-				    struct device_header *dev_hdr,
-				    void (*generate_irq_prop)(void *fdt,
-							      u8 irq,
-							      enum irq_type))
+static void plic__generate_fdt_node(void *fdt, struct kvm *kvm)
 {
 	u32 i;
+	char name[64];
 	u32 reg_cells[4], *irq_cells;
 
 	reg_cells[0] = 0;
-	reg_cells[1] = cpu_to_fdt32(RISCV_PLIC);
+	reg_cells[1] = cpu_to_fdt32(RISCV_IRQCHIP);
 	reg_cells[2] = 0;
-	reg_cells[3] = cpu_to_fdt32(RISCV_PLIC_SIZE);
+	reg_cells[3] = cpu_to_fdt32(RISCV_IRQCHIP_SIZE);
 
 	irq_cells = calloc(plic.num_context * 2, sizeof(u32));
 	if (!irq_cells)
 		die("Failed to alloc irq_cells");
 
-	_FDT(fdt_begin_node(fdt, "interrupt-controller at 0c000000"));
+	sprintf(name, "interrupt-controller@%08x", (u32)RISCV_IRQCHIP);
+	_FDT(fdt_begin_node(fdt, name));
 	_FDT(fdt_property_string(fdt, "compatible", "riscv,plic0"));
 	_FDT(fdt_property(fdt, "reg", reg_cells, sizeof(reg_cells)));
 	_FDT(fdt_property_cell(fdt, "#interrupt-cells", 1));
@@ -518,12 +506,10 @@ static int plic__init(struct kvm *kvm)
 	int ret;
 	struct plic_context *c;
 
-	plic.kvm = kvm;
-	plic.dev_hdr = (struct device_header) {
-		.bus_type	= DEVICE_BUS_MMIO,
-		.data		= plic__generate_fdt_node,
-	};
+	if (riscv_irqchip != IRQCHIP_PLIC)
+		return 0;
 
+	plic.kvm = kvm;
 	plic.num_irq = MAX_DEVICES;
 	plic.num_irq_word = plic.num_irq / 32;
 	if ((plic.num_irq_word * 32) < plic.num_irq)
@@ -544,15 +530,11 @@ static int plic__init(struct kvm *kvm)
 
 	mutex_init(&plic.irq_lock);
 

riscv/plic.c static int plic__init(struct kvm *kvm) · line 555

Quoted 10 lines
riscv/plic.c +1 -5
-	ret = kvm__register_mmio(kvm, RISCV_PLIC, RISCV_PLIC_SIZE,
+	ret = kvm__register_mmio(kvm, RISCV_IRQCHIP, RISCV_IRQCHIP_SIZE,
 				 false, plic__mmio_callback, &plic);
 	if (ret)
 		return ret;
 
-	ret = device__register(&plic.dev_hdr);
-	if (ret)
-		return ret;
-

Dropping this device__register() made me scratch my head a bit. I think it's not necessary to enumerate the irqchip in the device list and its fdt node is now generated by riscv_irqchip_generate_fdt_node(), but it'd be a lot easier for me if the commit message explained why this is OK.

Quoted 35 lines
riscv/plic.c +18 -1
 	plic.ready = true;
 
 	return 0;
@@ -562,10 +544,27 @@ dev_init(plic__init);
 
 static int plic__exit(struct kvm *kvm)
 {
+	if (riscv_irqchip != IRQCHIP_PLIC)
+		return 0;
+
 	plic.ready = false;
-	kvm__deregister_mmio(kvm, RISCV_PLIC);
+	kvm__deregister_mmio(kvm, RISCV_IRQCHIP);
 	free(plic.contexts);
 
 	return 0;
 }
 dev_exit(plic__exit);
+
+void plic__create(struct kvm *kvm)
+{
+	if (riscv_irqchip != IRQCHIP_UNKNOWN)
+		return;
+
+	riscv_irqchip = IRQCHIP_PLIC;
+	riscv_irqchip_inkernel = false;
+	riscv_irqchip_trigger = plic__irq_trig;
+	riscv_irqchip_generate_fdt_node = plic__generate_fdt_node;
+	riscv_irqchip_phandle = PHANDLE_PLIC;
+	riscv_irqchip_msi_phandle = PHANDLE_RESERVED;
+	riscv_irqchip_line_sensing = false;
+}
Signature · 1 line
2.34.1

Besides the nits, the whitespace issue, and request for another sentence in the commit message,

Thanks,
drew

author
#6 lore raw

Re: [kvmtool PATCH v2 3/6] riscv: Make irqchip support pluggable

Quoted 128 lines · On Wed, Oct 25, 2023 at 6:40?PM Andrew Jones <[email protected]> wrote:
Quoted 52 lines · Anup Patel

We will be having different types of irqchip:
1) PLIC emulated by user-space
2) AIA APLIC and IMSIC provided by in-kernel KVM module

To support above, we de-couple PLIC specific code from generic RISC-V code (such as FDT generation) so that we can easily add other types of irqchip.

---
 riscv/fdt.c                  | 14 ++++++--
 riscv/include/kvm/kvm-arch.h | 25 ++++++++++++---
 riscv/irq.c                  | 62 ++++++++++++++++++++++++++++++++++--
 riscv/kvm.c                  |  2 ++
 riscv/pci.c                  | 32 +++++++++++++------
 riscv/plic.c                 | 61 +++++++++++++++++------------------
 6 files changed, 147 insertions(+), 49 deletions(-)
riscv/fdt.c +9 -2
index 2724c6e..9af71b5 100644
@@ -209,19 +209,26 @@ static int setup_fdt(struct kvm *kvm)
      /* CPUs */
      generate_cpu_nodes(fdt, kvm);
 
+     /* IRQCHIP */
+     if (!riscv_irqchip_generate_fdt_node)
+             die("No way to generate IRQCHIP FDT node\n");
+     riscv_irqchip_generate_fdt_node(fdt, kvm);
+
      /* Simple Bus */
      _FDT(fdt_begin_node(fdt, "smb"));
      _FDT(fdt_property_string(fdt, "compatible", "simple-bus"));
      _FDT(fdt_property_cell(fdt, "#address-cells", 0x2));
      _FDT(fdt_property_cell(fdt, "#size-cells", 0x2));
-     _FDT(fdt_property_cell(fdt, "interrupt-parent", PHANDLE_PLIC));
+     _FDT(fdt_property_cell(fdt, "interrupt-parent",
+                            riscv_irqchip_phandle));
      _FDT(fdt_property(fdt, "ranges", NULL, 0));
 
      /* Virtio MMIO devices */
      dev_hdr = device__first_dev(DEVICE_BUS_MMIO);
      while (dev_hdr) {
              generate_mmio_fdt_nodes = dev_hdr->data;
-             generate_mmio_fdt_nodes(fdt, dev_hdr, plic__generate_irq_prop);
+             generate_mmio_fdt_nodes(fdt, dev_hdr,
+                                     riscv__generate_irq_prop);
              dev_hdr = device__next_dev(dev_hdr);
      }
 

riscv/fdt.c static int setup_fdt(struct kvm *kvm) · line 242

Quoted 8 lines
riscv/fdt.c +2 -1
@@ -229,7 +236,8 @@ static int setup_fdt(struct kvm *kvm)
      dev_hdr = device__first_dev(DEVICE_BUS_IOPORT);
      while (dev_hdr) {
              generate_mmio_fdt_nodes = dev_hdr->data;
-             generate_mmio_fdt_nodes(fdt, dev_hdr, plic__generate_irq_prop);
+             generate_mmio_fdt_nodes(fdt, dev_hdr,
+                                     riscv__generate_irq_prop);
              dev_hdr = device__next_dev(dev_hdr);
      }

nit: I'd let the above lines run long, but I know you prefer wrapping

     at 80.

riscv/include/kvm/kvm-arch.h line 14

Quoted 13 lines
riscv/include/kvm/kvm-arch.h +2 -2
index 660355b..cd37fc6 100644
@@ -10,8 +10,8 @@
 
 #define RISCV_IOPORT         0x00000000ULL
 #define RISCV_IOPORT_SIZE    SZ_64K
-#define RISCV_PLIC           0x0c000000ULL
-#define RISCV_PLIC_SIZE              SZ_64M
+#define RISCV_IRQCHIP                0x08000000ULL
+#define RISCV_IRQCHIP_SIZE           SZ_128M

I checked the applied patch and the SZ_128M is over indented.

Quoted 46 lines
riscv/include/kvm/kvm-arch.h +19 -2
 #define RISCV_MMIO           0x10000000ULL
 #define RISCV_MMIO_SIZE              SZ_512M
 #define RISCV_PCI            0x30000000ULL
@@ -84,10 +84,27 @@ static inline bool riscv_addr_in_ioport_region(u64 phys_addr)
 
 enum irq_type;
 
-void plic__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type);
+enum irqchip_type {
+     IRQCHIP_UNKNOWN = 0,
+     IRQCHIP_PLIC,
+     IRQCHIP_AIA
+};
+
+extern enum irqchip_type riscv_irqchip;
+extern bool riscv_irqchip_inkernel;
+extern void (*riscv_irqchip_trigger)(struct kvm *kvm, int irq,
+                                  int level, bool edge);
+extern void (*riscv_irqchip_generate_fdt_node)(void *fdt, struct kvm *kvm);
+extern u32 riscv_irqchip_phandle;
+extern u32 riscv_irqchip_msi_phandle;
+extern bool riscv_irqchip_line_sensing;
 
-void plic__irq_trig(struct kvm *kvm, int irq, int level, bool edge);
+void plic__create(struct kvm *kvm);
 
 void pci__generate_fdt_nodes(void *fdt);
 
+void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type);
+
+void riscv__irqchip_create(struct kvm *kvm);
+
 #endif /* KVM__KVM_ARCH_H */
riscv/irq.c +5 -0
index 78a582d..b608a2f 100644
@@ -1,13 +1,71 @@
 #include "kvm/kvm.h"
 #include "kvm/kvm-cpu.h"
 #include "kvm/irq.h"
+#include "kvm/fdt.h"
+#include "kvm/virtio.h"
+
+enum irqchip_type riscv_irqchip = IRQCHIP_UNKNOWN;
+bool riscv_irqchip_inkernel = false;
Quoted 8 lines

riscv/irq.c line 14

Quoted 6 lines
riscv/irq.c +6 -0
+void (*riscv_irqchip_trigger)(struct kvm *kvm, int irq, int level, bool edge)
+                             = NULL;
+void (*riscv_irqchip_generate_fdt_node)(void *fdt, struct kvm *kvm) = NULL;
+u32 riscv_irqchip_phandle = PHANDLE_RESERVED;
+u32 riscv_irqchip_msi_phandle = PHANDLE_RESERVED;
+bool riscv_irqchip_line_sensing = false;

nit: no need to init BSS symbols with zeros.

Okay, I will update.

Quoted 126 lines
Quoted 126 lines
riscv/irq.c +49 -2
 void kvm__irq_line(struct kvm *kvm, int irq, int level)
 {
-     plic__irq_trig(kvm, irq, level, false);
+     struct kvm_irq_level irq_level;
+
+     if (riscv_irqchip_inkernel) {
+             irq_level.irq = irq;
+             irq_level.level = !!level;
+             if (ioctl(kvm->vm_fd, KVM_IRQ_LINE, &irq_level) < 0)
+                     pr_warning("%s: Could not KVM_IRQ_LINE for irq %d\n",
+                                __func__, irq);
+     } else {
+             if (riscv_irqchip_trigger)
+                     riscv_irqchip_trigger(kvm, irq, level, false);
+             else
+                     pr_warning("%s: Can't change level for irq %d\n",
+                                __func__, irq);
+     }
 }
 
 void kvm__irq_trigger(struct kvm *kvm, int irq)
 {
-     plic__irq_trig(kvm, irq, 1, true);
+     if (riscv_irqchip_inkernel) {
+             kvm__irq_line(kvm, irq, VIRTIO_IRQ_HIGH);
+             kvm__irq_line(kvm, irq, VIRTIO_IRQ_LOW);
+     } else {
+             if (riscv_irqchip_trigger)
+                     riscv_irqchip_trigger(kvm, irq, 1, true);
+             else
+                     pr_warning("%s: Can't trigger irq %d\n",
+                                __func__, irq);
+     }
+}
+
+void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type)
+{
+     u32 prop[2], size;
+
+     prop[0] = cpu_to_fdt32(irq);
+     size = sizeof(u32);
+     if (riscv_irqchip_line_sensing) {
+             prop[1] = cpu_to_fdt32(irq_type);
+             size += sizeof(u32);
+     }
+
+     _FDT(fdt_property(fdt, "interrupts", prop, size));
+}
+
+void riscv__irqchip_create(struct kvm *kvm)
+{
+     /* Try PLIC irqchip */
+     plic__create(kvm);
+
+     /* Fail if irqchip unknown */
+     if (riscv_irqchip == IRQCHIP_UNKNOWN)
+             die("No IRQCHIP found\n");
 }
riscv/kvm.c +2 -0
index 8daad94..1d49479 100644
@@ -96,6 +96,8 @@ void kvm__arch_init(struct kvm *kvm)
 
      madvise(kvm->arch.ram_alloc_start, kvm->arch.ram_alloc_size,
              MADV_HUGEPAGE);
+
+     riscv__irqchip_create(kvm);
 }
 
 #define FDT_ALIGN    SZ_4M
riscv/pci.c +11 -5
index 604fd20..61dee06 100644
@@ -7,20 +7,21 @@
 
 /*
  * An entry in the interrupt-map table looks like:
- * <pci unit address> <pci interrupt pin> <plic phandle> <plic interrupt>
+ * <pci unit address> <pci interrupt pin> <irqchip phandle> <irqchip line>
  */
 
 struct of_interrupt_map_entry {
      struct of_pci_irq_mask          pci_irq_mask;
-     u32                             plic_phandle;
-     u32                             plic_irq;
+     u32                             irqchip_phandle;
+     u32                             irqchip_line;
+     u32                             irqchip_sense;
 } __attribute__((packed));
 
 void pci__generate_fdt_nodes(void *fdt)
 {
      struct device_header *dev_hdr;
      struct of_interrupt_map_entry irq_map[OF_PCI_IRQ_MAP_MAX];
-     unsigned nentries = 0;
+     unsigned nentries = 0, nsize;
      /* Bus range */
      u32 bus_range[] = { cpu_to_fdt32(0), cpu_to_fdt32(1), };
      /* Configuration Space */
@@ -48,6 +49,11 @@ void pci__generate_fdt_nodes(void *fdt)
              },
      };
 
+     /* Find size of each interrupt map entery */
+     nsize = sizeof(struct of_interrupt_map_entry);
+     if (!riscv_irqchip_line_sensing)
+             nsize -= sizeof(u32);
+
      /* Boilerplate PCI properties */
      _FDT(fdt_begin_node(fdt, "pci"));
      _FDT(fdt_property_string(fdt, "device_type", "pci"));
@@ -64,12 +70,13 @@ void pci__generate_fdt_nodes(void *fdt)
      /* Generate the interrupt map ... */
      dev_hdr = device__first_dev(DEVICE_BUS_PCI);
      while (dev_hdr && nentries < ARRAY_SIZE(irq_map)) {
-             struct of_interrupt_map_entry *entry = &irq_map[nentries];
+             struct of_interrupt_map_entry *entry;
              struct pci_device_header *pci_hdr = dev_hdr->data;
              u8 dev_num = dev_hdr->dev_num;
              u8 pin = pci_hdr->irq_pin;
              u8 irq = pci_hdr->irq_line;
 
Quoted 3 lines

riscv/pci.c void pci__generate_fdt_nodes(void *fdt) · line 79

Quoted 1 line
riscv/pci.c +1 -0
+             entry = ((void *)irq_map) + (nsize * nentries);

nit: The outer () of both terms are superfluous.

Okay, I will update.

Quoted 135 lines
Quoted 135 lines
riscv/pci.c +11 -4
              *entry = (struct of_interrupt_map_entry) {
                      .pci_irq_mask = {
                              .pci_addr = {
@@ -79,16 +86,18 @@ void pci__generate_fdt_nodes(void *fdt)
                              },
                              .pci_pin        = cpu_to_fdt32(pin),
                      },
-                     .plic_phandle   = cpu_to_fdt32(PHANDLE_PLIC),
-                     .plic_irq       = cpu_to_fdt32(irq),
+                     .irqchip_phandle        = cpu_to_fdt32(riscv_irqchip_phandle),
+                     .irqchip_line           = cpu_to_fdt32(irq),
              };
 
+             if (riscv_irqchip_line_sensing)
+                     entry->irqchip_sense = cpu_to_fdt32(IRQ_TYPE_LEVEL_HIGH);
+
              nentries++;
              dev_hdr = device__next_dev(dev_hdr);
      }
 
-     _FDT(fdt_property(fdt, "interrupt-map", irq_map,
-                       sizeof(struct of_interrupt_map_entry) * nentries));
+     _FDT(fdt_property(fdt, "interrupt-map", irq_map, nsize * nentries));
 
      /* ... and the corresponding mask. */
      if (nentries) {
@@ -105,5 +114,10 @@ void pci__generate_fdt_nodes(void *fdt)
                                sizeof(irq_mask)));
      }
 
+     /* Set MSI parent if available */
+     if (riscv_irqchip_msi_phandle != PHANDLE_RESERVED)
+             _FDT(fdt_property_cell(fdt, "msi-parent",
+                                    riscv_irqchip_msi_phandle));
+
      _FDT(fdt_end_node(fdt));
 }
riscv/plic.c +12 -26
index 6242286..ab7c574 100644
@@ -118,7 +118,6 @@ struct plic_context {
 struct plic_state {
      bool ready;
      struct kvm *kvm;
-     struct device_header dev_hdr;
 
      /* Static Configuration */
      u32 num_irq;
@@ -204,7 +203,7 @@ static u32 __plic_context_irq_claim(struct plic_state *s,
      return best_irq;
 }
 
-void plic__irq_trig(struct kvm *kvm, int irq, int level, bool edge)
+static void plic__irq_trig(struct kvm *kvm, int irq, int level, bool edge)
 {
      bool irq_marked = false;
      u8 i, irq_prio, irq_word;
@@ -425,7 +424,7 @@ static void plic__mmio_callback(struct kvm_cpu *vcpu,
              die("plic: invalid len=%d", len);
 
      addr &= ~0x3;
-     addr -= RISCV_PLIC;
+     addr -= RISCV_IRQCHIP;
 
      if (is_write) {
              if (PRIORITY_BASE <= addr && addr < ENABLE_BASE) {
@@ -464,34 +463,23 @@ static void plic__mmio_callback(struct kvm_cpu *vcpu,
      }
 }
 
-void plic__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type)
-{
-     u32 irq_prop[] = {
-             cpu_to_fdt32(irq)
-     };
-
-     _FDT(fdt_property(fdt, "interrupts", irq_prop, sizeof(irq_prop)));
-}
-
-static void plic__generate_fdt_node(void *fdt,
-                                 struct device_header *dev_hdr,
-                                 void (*generate_irq_prop)(void *fdt,
-                                                           u8 irq,
-                                                           enum irq_type))
+static void plic__generate_fdt_node(void *fdt, struct kvm *kvm)
 {
      u32 i;
+     char name[64];
      u32 reg_cells[4], *irq_cells;
 
      reg_cells[0] = 0;
-     reg_cells[1] = cpu_to_fdt32(RISCV_PLIC);
+     reg_cells[1] = cpu_to_fdt32(RISCV_IRQCHIP);
      reg_cells[2] = 0;
-     reg_cells[3] = cpu_to_fdt32(RISCV_PLIC_SIZE);
+     reg_cells[3] = cpu_to_fdt32(RISCV_IRQCHIP_SIZE);
 
      irq_cells = calloc(plic.num_context * 2, sizeof(u32));
      if (!irq_cells)
              die("Failed to alloc irq_cells");
 
-     _FDT(fdt_begin_node(fdt, "interrupt-controller at 0c000000"));
+     sprintf(name, "interrupt-controller@%08x", (u32)RISCV_IRQCHIP);
+     _FDT(fdt_begin_node(fdt, name));
      _FDT(fdt_property_string(fdt, "compatible", "riscv,plic0"));
      _FDT(fdt_property(fdt, "reg", reg_cells, sizeof(reg_cells)));
      _FDT(fdt_property_cell(fdt, "#interrupt-cells", 1));
@@ -518,12 +506,10 @@ static int plic__init(struct kvm *kvm)
      int ret;
      struct plic_context *c;
 
-     plic.kvm = kvm;
-     plic.dev_hdr = (struct device_header) {
-             .bus_type       = DEVICE_BUS_MMIO,
-             .data           = plic__generate_fdt_node,
-     };
+     if (riscv_irqchip != IRQCHIP_PLIC)
+             return 0;
 
+     plic.kvm = kvm;
      plic.num_irq = MAX_DEVICES;
      plic.num_irq_word = plic.num_irq / 32;
      if ((plic.num_irq_word * 32) < plic.num_irq)
@@ -544,15 +530,11 @@ static int plic__init(struct kvm *kvm)
 
      mutex_init(&plic.irq_lock);
 
-     ret = kvm__register_mmio(kvm, RISCV_PLIC, RISCV_PLIC_SIZE,
+     ret = kvm__register_mmio(kvm, RISCV_IRQCHIP, RISCV_IRQCHIP_SIZE,
                               false, plic__mmio_callback, &plic);
      if (ret)
              return ret;
 
Quoted 9 lines

riscv/plic.c static int plic__init(struct kvm *kvm) · line 555

Quoted 4 lines
riscv/plic.c +0 -4
-     ret = device__register(&plic.dev_hdr);
-     if (ret)
-             return ret;
-

Dropping this device__register() made me scratch my head a bit. I think it's not necessary to enumerate the irqchip in the device list and its fdt node is now generated by riscv_irqchip_generate_fdt_node(), but it'd be a lot easier for me if the commit message explained why this is OK.

Okay, I will update the commit description.

Quoted 40 lines
Quoted 34 lines
riscv/plic.c +18 -2
      plic.ready = true;
 
      return 0;
@@ -562,10 +544,27 @@ dev_init(plic__init);
 
 static int plic__exit(struct kvm *kvm)
 {
+     if (riscv_irqchip != IRQCHIP_PLIC)
+             return 0;
+
      plic.ready = false;
-     kvm__deregister_mmio(kvm, RISCV_PLIC);
+     kvm__deregister_mmio(kvm, RISCV_IRQCHIP);
      free(plic.contexts);
 
      return 0;
 }
 dev_exit(plic__exit);
+
+void plic__create(struct kvm *kvm)
+{
+     if (riscv_irqchip != IRQCHIP_UNKNOWN)
+             return;
+
+     riscv_irqchip = IRQCHIP_PLIC;
+     riscv_irqchip_inkernel = false;
+     riscv_irqchip_trigger = plic__irq_trig;
+     riscv_irqchip_generate_fdt_node = plic__generate_fdt_node;
+     riscv_irqchip_phandle = PHANDLE_PLIC;
+     riscv_irqchip_msi_phandle = PHANDLE_RESERVED;
+     riscv_irqchip_line_sensing = false;
+}
--
Quoted 2 lines

2.34.1

Besides the nits, the whitespace issue, and request for another sentence in the commit message,

Quoted 4 lines

Thanks,
drew

Regards,
Anup

author
#7 lore raw

PATCH v2 4/6 riscv: Add IRQFD support for in-kernel AIA irqchip

In reply to Anup Patel

To use irqfd with in-kernel AIA irqchip, we add custom irq__add_irqfd and irq__del_irqfd functions. This allows us to defer actual KVM_IRQFD ioctl() until AIA irqchip is initialized by KVMTOOL.

---
 riscv/include/kvm/kvm-arch.h | 11 ++++++
 riscv/irq.c                  | 73 ++++++++++++++++++++++++++++++++++++
 2 files changed, 84 insertions(+)
riscv/include/kvm/kvm-arch.h +11 -0
index cd37fc6..1a8af6a 100644
@@ -98,11 +98,22 @@ extern void (*riscv_irqchip_generate_fdt_node)(void *fdt, struct kvm *kvm);
 extern u32 riscv_irqchip_phandle;
 extern u32 riscv_irqchip_msi_phandle;
 extern bool riscv_irqchip_line_sensing;
+extern bool riscv_irqchip_irqfd_ready;
 
 void plic__create(struct kvm *kvm);
 
 void pci__generate_fdt_nodes(void *fdt);
 
+int riscv__add_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd,
+		     int resample_fd);
+
+void riscv__del_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd);
+
+#define irq__add_irqfd riscv__add_irqfd
+#define irq__del_irqfd riscv__del_irqfd
+
+int riscv__setup_irqfd_lines(struct kvm *kvm);
+
 void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type);
 
 void riscv__irqchip_create(struct kvm *kvm);
riscv/irq.c +73 -0

85 lines not on this page · raw message

Signature · 1 line
2.34.1
Reviewed-by on v2 4/6
#8 lore raw

Re: [kvmtool PATCH v2 4/6] riscv: Add IRQFD support for in-kernel AIA irqchip

Quoted 63 lines · Anup Patel

To use irqfd with in-kernel AIA irqchip, we add custom irq__add_irqfd and irq__del_irqfd functions. This allows us to defer actual KVM_IRQFD ioctl() until AIA irqchip is initialized by KVMTOOL.

---
 riscv/include/kvm/kvm-arch.h | 11 ++++++
 riscv/irq.c                  | 73 ++++++++++++++++++++++++++++++++++++
 2 files changed, 84 insertions(+)
riscv/include/kvm/kvm-arch.h +11 -0
index cd37fc6..1a8af6a 100644
@@ -98,11 +98,22 @@ extern void (*riscv_irqchip_generate_fdt_node)(void *fdt, struct kvm *kvm);
 extern u32 riscv_irqchip_phandle;
 extern u32 riscv_irqchip_msi_phandle;
 extern bool riscv_irqchip_line_sensing;
+extern bool riscv_irqchip_irqfd_ready;
 
 void plic__create(struct kvm *kvm);
 
 void pci__generate_fdt_nodes(void *fdt);
 
+int riscv__add_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd,
+		     int resample_fd);
+
+void riscv__del_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd);
+
+#define irq__add_irqfd riscv__add_irqfd
+#define irq__del_irqfd riscv__del_irqfd
+
+int riscv__setup_irqfd_lines(struct kvm *kvm);
+
 void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type);
 
 void riscv__irqchip_create(struct kvm *kvm);
riscv/irq.c +10 -0
index b608a2f..e6c0939 100644
@@ -12,6 +12,7 @@ void (*riscv_irqchip_generate_fdt_node)(void *fdt, struct kvm *kvm) = NULL;
 u32 riscv_irqchip_phandle = PHANDLE_RESERVED;
 u32 riscv_irqchip_msi_phandle = PHANDLE_RESERVED;
 bool riscv_irqchip_line_sensing = false;
+bool riscv_irqchip_irqfd_ready = false;
 
 void kvm__irq_line(struct kvm *kvm, int irq, int level)
 {
@@ -46,6 +47,78 @@ void kvm__irq_trigger(struct kvm *kvm, int irq)
 	}
 }
 
+struct riscv_irqfd_line {
+	unsigned int		gsi;
+	int			trigger_fd;
+	int			resample_fd;
+	struct list_head	list;
+};
+
+static LIST_HEAD(irqfd_lines);
+

riscv/irq.c void kvm__irq_trigger(struct kvm *kvm, int irq) · line 68

Quoted 10 lines
riscv/irq.c +10 -0
+int riscv__add_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd,
+		     int resample_fd)
+{
+	struct riscv_irqfd_line *line;
+
+	if (riscv_irqchip_irqfd_ready)
+		return irq__common_add_irqfd(kvm, gsi, trigger_fd,
+					     resample_fd);
+
+	/* Postpone the routing setup until we have a distributor */

This comment comes from the Arm code. We probably want to replace distributor with "until the AIA is initialized" or something.

Quoted 59 lines
riscv/irq.c +53 -0
+	line = malloc(sizeof(*line));
+	if (!line)
+		return -ENOMEM;
+
+	*line = (struct riscv_irqfd_line) {
+		.gsi		= gsi,
+		.trigger_fd	= trigger_fd,
+		.resample_fd	= resample_fd,
+	};
+	list_add(&line->list, &irqfd_lines);
+
+	return 0;
+}
+
+void riscv__del_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd)
+{
+	struct riscv_irqfd_line *line;
+
+	if (riscv_irqchip_irqfd_ready) {
+		irq__common_del_irqfd(kvm, gsi, trigger_fd);
+		return;
+	}
+
+	list_for_each_entry(line, &irqfd_lines, list) {
+		if (line->gsi != gsi)
+			continue;
+
+		list_del(&line->list);
+		free(line);
+		break;
+	}
+}
+
+int riscv__setup_irqfd_lines(struct kvm *kvm)
+{
+	int ret;
+	struct riscv_irqfd_line *line, *tmp;
+
+	list_for_each_entry_safe(line, tmp, &irqfd_lines, list) {
+		ret = irq__common_add_irqfd(kvm, line->gsi, line->trigger_fd,
+					    line->resample_fd);
+		if (ret < 0) {
+			pr_err("Failed to register IRQFD");
+			return ret;
+		}
+
+		list_del(&line->list);
+		free(line);
+	}
+
+	return 0;
+}
+
 void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type)
 {
 	u32 prop[2], size;
Signature · 1 line
2.34.1

Other than the comment,

author
#9 lore raw

Re: [kvmtool PATCH v2 4/6] riscv: Add IRQFD support for in-kernel AIA irqchip

Quoted 70 lines · On Wed, Oct 25, 2023 at 6:47?PM Andrew Jones <[email protected]> wrote:
Quoted 68 lines · Anup Patel

To use irqfd with in-kernel AIA irqchip, we add custom irq__add_irqfd and irq__del_irqfd functions. This allows us to defer actual KVM_IRQFD ioctl() until AIA irqchip is initialized by KVMTOOL.

---
 riscv/include/kvm/kvm-arch.h | 11 ++++++
 riscv/irq.c                  | 73 ++++++++++++++++++++++++++++++++++++
 2 files changed, 84 insertions(+)
riscv/include/kvm/kvm-arch.h +11 -0
index cd37fc6..1a8af6a 100644
@@ -98,11 +98,22 @@ extern void (*riscv_irqchip_generate_fdt_node)(void *fdt, struct kvm *kvm);
 extern u32 riscv_irqchip_phandle;
 extern u32 riscv_irqchip_msi_phandle;
 extern bool riscv_irqchip_line_sensing;
+extern bool riscv_irqchip_irqfd_ready;
 
 void plic__create(struct kvm *kvm);
 
 void pci__generate_fdt_nodes(void *fdt);
 
+int riscv__add_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd,
+                  int resample_fd);
+
+void riscv__del_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd);
+
+#define irq__add_irqfd riscv__add_irqfd
+#define irq__del_irqfd riscv__del_irqfd
+
+int riscv__setup_irqfd_lines(struct kvm *kvm);
+
 void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type);
 
 void riscv__irqchip_create(struct kvm *kvm);
riscv/irq.c +15 -0
index b608a2f..e6c0939 100644
@@ -12,6 +12,7 @@ void (*riscv_irqchip_generate_fdt_node)(void *fdt, struct kvm *kvm) = NULL;
 u32 riscv_irqchip_phandle = PHANDLE_RESERVED;
 u32 riscv_irqchip_msi_phandle = PHANDLE_RESERVED;
 bool riscv_irqchip_line_sensing = false;
+bool riscv_irqchip_irqfd_ready = false;
 
 void kvm__irq_line(struct kvm *kvm, int irq, int level)
 {
@@ -46,6 +47,78 @@ void kvm__irq_trigger(struct kvm *kvm, int irq)
      }
 }
 
+struct riscv_irqfd_line {
+     unsigned int            gsi;
+     int                     trigger_fd;
+     int                     resample_fd;
+     struct list_head        list;
+};
+
+static LIST_HEAD(irqfd_lines);
+
+int riscv__add_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd,
+                  int resample_fd)
+{
+     struct riscv_irqfd_line *line;
+
Quoted 8 lines

riscv/irq.c void kvm__irq_trigger(struct kvm *kvm, int irq) · line 68

Quoted 5 lines
riscv/irq.c +5 -0
+     if (riscv_irqchip_irqfd_ready)
+             return irq__common_add_irqfd(kvm, gsi, trigger_fd,
+                                          resample_fd);
+
+     /* Postpone the routing setup until we have a distributor */

This comment comes from the Arm code. We probably want to replace distributor with "until the AIA is initialized" or something.

Okay, I will update.

Quoted 63 lines
Quoted 58 lines
riscv/irq.c +53 -1
+     line = malloc(sizeof(*line));
+     if (!line)
+             return -ENOMEM;
+
+     *line = (struct riscv_irqfd_line) {
+             .gsi            = gsi,
+             .trigger_fd     = trigger_fd,
+             .resample_fd    = resample_fd,
+     };
+     list_add(&line->list, &irqfd_lines);
+
+     return 0;
+}
+
+void riscv__del_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd)
+{
+     struct riscv_irqfd_line *line;
+
+     if (riscv_irqchip_irqfd_ready) {
+             irq__common_del_irqfd(kvm, gsi, trigger_fd);
+             return;
+     }
+
+     list_for_each_entry(line, &irqfd_lines, list) {
+             if (line->gsi != gsi)
+                     continue;
+
+             list_del(&line->list);
+             free(line);
+             break;
+     }
+}
+
+int riscv__setup_irqfd_lines(struct kvm *kvm)
+{
+     int ret;
+     struct riscv_irqfd_line *line, *tmp;
+
+     list_for_each_entry_safe(line, tmp, &irqfd_lines, list) {
+             ret = irq__common_add_irqfd(kvm, line->gsi, line->trigger_fd,
+                                         line->resample_fd);
+             if (ret < 0) {
+                     pr_err("Failed to register IRQFD");
+                     return ret;
+             }
+
+             list_del(&line->list);
+             free(line);
+     }
+
+     return 0;
+}
+
 void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type)
 {
      u32 prop[2], size;
--
Quoted 2 lines

2.34.1

Other than the comment,

Quoted 1 line

Regards,
Anup

author
#10 lore raw

PATCH v2 5/6 riscv: Use AIA in-kernel irqchip whenever KVM RISC-V supports

In reply to Anup Patel

The KVM RISC-V kernel module supports AIA in-kernel irqchip when underlying host has AIA support. We detect and use AIA in-kernel irqchip whenever possible otherwise we fallback to PLIC emulated in user-space.

---
 Makefile                     |   1 +
 riscv/aia.c                  | 227 +++++++++++++++++++++++++++++++++++
 riscv/include/kvm/fdt-arch.h |   8 +-
 riscv/include/kvm/kvm-arch.h |   2 +
 riscv/irq.c                  |   3 +
 5 files changed, 240 insertions(+), 1 deletion(-)
 create mode 100644 riscv/aia.c
Makefile +1 -0
index e711670..acd5ffd 100644
@@ -220,6 +220,7 @@ ifeq ($(ARCH),riscv)
 	OBJS		+= riscv/kvm-cpu.o
 	OBJS		+= riscv/pci.o
 	OBJS		+= riscv/plic.o
+	OBJS		+= riscv/aia.o
 	ifeq ($(RISCV_XLEN),32)
 		CFLAGS	+= -mabi=ilp32d -march=rv32gc
 	endif
riscv/aia.c new file +227 -0

227 lines not on this page · raw message

riscv/include/kvm/fdt-arch.h +7 -1
index f7548e8..d88b832 100644
@@ -1,7 +1,13 @@
 #ifndef KVM__KVM_FDT_H
 #define KVM__KVM_FDT_H
 
-enum phandles {PHANDLE_RESERVED = 0, PHANDLE_PLIC, PHANDLES_MAX};
+enum phandles {
+	PHANDLE_RESERVED = 0,
+	PHANDLE_PLIC,
+	PHANDLE_AIA_APLIC,
+	PHANDLE_AIA_IMSIC,
+	PHANDLES_MAX
+};
 
 #define PHANDLE_CPU_INTC_BASE	PHANDLES_MAX
 
riscv/include/kvm/kvm-arch.h +2 -0
index 1a8af6a..9f2159f 100644
@@ -100,6 +100,8 @@ extern u32 riscv_irqchip_msi_phandle;
 extern bool riscv_irqchip_line_sensing;
 extern bool riscv_irqchip_irqfd_ready;
 
+void aia__create(struct kvm *kvm);
+
 void plic__create(struct kvm *kvm);
 
 void pci__generate_fdt_nodes(void *fdt);
riscv/irq.c +3 -0
index e6c0939..be3e7ac 100644
@@ -135,6 +135,9 @@ void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type)
 
 void riscv__irqchip_create(struct kvm *kvm)
 {
+	/* Try AIA in-kernel irqchip. */
+	aia__create(kvm);
+
 	/* Try PLIC irqchip */
 	plic__create(kvm);
 
Signature · 1 line
2.34.1
Reviewed-by on v2 5/6
#11 lore raw

Re: [kvmtool PATCH v2 5/6] riscv: Use AIA in-kernel irqchip whenever KVM RISC-V supports

Quoted 69 lines · Anup Patel

The KVM RISC-V kernel module supports AIA in-kernel irqchip when underlying host has AIA support. We detect and use AIA in-kernel irqchip whenever possible otherwise we fallback to PLIC emulated in user-space.

---
 Makefile                     |   1 +
 riscv/aia.c                  | 227 +++++++++++++++++++++++++++++++++++
 riscv/include/kvm/fdt-arch.h |   8 +-
 riscv/include/kvm/kvm-arch.h |   2 +
 riscv/irq.c                  |   3 +
 5 files changed, 240 insertions(+), 1 deletion(-)
 create mode 100644 riscv/aia.c
Makefile +1 -0
index e711670..acd5ffd 100644
@@ -220,6 +220,7 @@ ifeq ($(ARCH),riscv)
 	OBJS		+= riscv/kvm-cpu.o
 	OBJS		+= riscv/pci.o
 	OBJS		+= riscv/plic.o
+	OBJS		+= riscv/aia.o
 	ifeq ($(RISCV_XLEN),32)
 		CFLAGS	+= -mabi=ilp32d -march=rv32gc
 	endif
riscv/aia.c new file +36 -0
new file mode 100644
index 0000000..8c85b3f
@@ -0,0 +1,227 @@
+#include "kvm/devices.h"
+#include "kvm/fdt.h"
+#include "kvm/ioeventfd.h"
+#include "kvm/ioport.h"
+#include "kvm/kvm.h"
+#include "kvm/kvm-cpu.h"
+#include "kvm/irq.h"
+#include "kvm/util.h"
+
+static int aia_fd = -1;
+
+static u32 aia_mode = KVM_DEV_RISCV_AIA_MODE_EMUL;
+static struct kvm_device_attr aia_mode_attr = {
+	.group	= KVM_DEV_RISCV_AIA_GRP_CONFIG,
+	.attr	= KVM_DEV_RISCV_AIA_CONFIG_MODE,
+};
+
+static u32 aia_nr_ids = 0;
+static struct kvm_device_attr aia_nr_ids_attr = {
+	.group	= KVM_DEV_RISCV_AIA_GRP_CONFIG,
+	.attr	= KVM_DEV_RISCV_AIA_CONFIG_IDS,
+};
+
+static u32 aia_nr_sources = 0;
+static struct kvm_device_attr aia_nr_sources_attr = {
+	.group	= KVM_DEV_RISCV_AIA_GRP_CONFIG,
+	.attr	= KVM_DEV_RISCV_AIA_CONFIG_SRCS,
+};
+
+static u32 aia_hart_bits = 0;
+static struct kvm_device_attr aia_hart_bits_attr = {
+	.group	= KVM_DEV_RISCV_AIA_GRP_CONFIG,
+	.attr	= KVM_DEV_RISCV_AIA_CONFIG_HART_BITS,
+};
+
+static u32 aia_nr_harts = 0;

riscv/aia.c line 46

Quoted 10 lines
riscv/aia.c new file +10 -0
+
+#define IRQCHIP_AIA_NR			0
+
+#define AIA_IMSIC_BASE			RISCV_IRQCHIP
+#define AIA_IMSIC_ADDR(__hart)		\
+	(AIA_IMSIC_BASE + (__hart) * KVM_DEV_RISCV_IMSIC_SIZE)
+#define AIA_IMSIC_SIZE			\
+	(aia_nr_harts * KVM_DEV_RISCV_IMSIC_SIZE)
+#define AIA_APLIC_ADDR(__nr_harts)	\
+	(AIA_IMSIC_BASE + (__nr_harts) * KVM_DEV_RISCV_IMSIC_SIZE)

AIA_APLIC_ADDR() probably doesn't need to take nr_harts since it's always called with aia_nr_harts. So it could just be defined as

#define AIA_APLIC_ADDR (AIA_IMSIC_BASE + AIA_IMSIC_SIZE)

Quoted 205 lines
riscv/aia.c new file +181 -0
+
+static void aia__generate_fdt_node(void *fdt, struct kvm *kvm)
+{
+	u32 i;
+	char name[64];
+	u32 reg_cells[4], *irq_cells;
+
+	irq_cells = calloc(aia_nr_harts * 2, sizeof(u32));
+	if (!irq_cells)
+		die("Failed to alloc irq_cells");
+
+	sprintf(name, "imsics@%08x", (u32)AIA_IMSIC_BASE);
+	_FDT(fdt_begin_node(fdt, name));
+	_FDT(fdt_property_string(fdt, "compatible", "riscv,imsics"));
+	reg_cells[0] = 0;
+	reg_cells[1] = cpu_to_fdt32(AIA_IMSIC_BASE);
+	reg_cells[2] = 0;
+	reg_cells[3] = cpu_to_fdt32(AIA_IMSIC_SIZE);
+	_FDT(fdt_property(fdt, "reg", reg_cells, sizeof(reg_cells)));
+	_FDT(fdt_property_cell(fdt, "#interrupt-cells", 0));
+	_FDT(fdt_property(fdt, "interrupt-controller", NULL, 0));
+	_FDT(fdt_property(fdt, "msi-controller", NULL, 0));
+	_FDT(fdt_property_cell(fdt, "riscv,num-ids", aia_nr_ids));
+	_FDT(fdt_property_cell(fdt, "phandle", PHANDLE_AIA_IMSIC));
+	for (i = 0; i < aia_nr_harts; i++) {
+		irq_cells[2*i + 0] = cpu_to_fdt32(PHANDLE_CPU_INTC_BASE + i);
+		irq_cells[2*i + 1] = cpu_to_fdt32(9);
+	}
+	_FDT(fdt_property(fdt, "interrupts-extended", irq_cells,
+			  sizeof(u32) * aia_nr_harts * 2));
+	_FDT(fdt_end_node(fdt));
+
+	free(irq_cells);
+
+	/* Skip APLIC node if we have no interrupt sources */
+	if (!aia_nr_sources)
+		return;
+
+	sprintf(name, "aplic@%08x", (u32)AIA_APLIC_ADDR(aia_nr_harts));
+	_FDT(fdt_begin_node(fdt, name));
+	_FDT(fdt_property_string(fdt, "compatible", "riscv,aplic"));
+	reg_cells[0] = 0;
+	reg_cells[1] = cpu_to_fdt32(AIA_APLIC_ADDR(aia_nr_harts));
+	reg_cells[2] = 0;
+	reg_cells[3] = cpu_to_fdt32(KVM_DEV_RISCV_APLIC_SIZE);
+	_FDT(fdt_property(fdt, "reg", reg_cells, sizeof(reg_cells)));
+	_FDT(fdt_property_cell(fdt, "#interrupt-cells", 2));
+	_FDT(fdt_property(fdt, "interrupt-controller", NULL, 0));
+	_FDT(fdt_property_cell(fdt, "riscv,num-sources", aia_nr_sources));
+	_FDT(fdt_property_cell(fdt, "phandle", PHANDLE_AIA_APLIC));
+	_FDT(fdt_property_cell(fdt, "msi-parent", PHANDLE_AIA_IMSIC));
+	_FDT(fdt_end_node(fdt));
+}
+
+static int aia__irq_routing_init(struct kvm *kvm)
+{
+	int r;
+	int irqlines = aia_nr_sources + 1;
+
+	/* Skip this if we have no interrupt sources */
+	if (!aia_nr_sources)
+		return 0;
+
+	/*
+	 * This describes the default routing that the kernel uses without
+	 * any routing explicitly set up via KVM_SET_GSI_ROUTING. So we
+	 * don't need to commit these setting right now. The first actual
+	 * user (MSI routing) will engage these mappings then.
+	 */
+	for (next_gsi = 0; next_gsi < irqlines; next_gsi++) {
+		r = irq__allocate_routing_entry();
+		if (r)
+			return r;
+
+		irq_routing->entries[irq_routing->nr++] =
+			(struct kvm_irq_routing_entry) {
+				.gsi = next_gsi,
+				.type = KVM_IRQ_ROUTING_IRQCHIP,
+				.u.irqchip.irqchip = IRQCHIP_AIA_NR,
+				.u.irqchip.pin = next_gsi,
+		};
+	}
+
+	return 0;
+}
+
+static int aia__init(struct kvm *kvm)
+{
+	int i, ret;
+	u64 aia_addr = 0;
+	struct kvm_device_attr aia_addr_attr = {
+		.group	= KVM_DEV_RISCV_AIA_GRP_ADDR,
+		.addr	= (u64)(unsigned long)&aia_addr,
+	};
+	struct kvm_device_attr aia_init_attr = {
+		.group	= KVM_DEV_RISCV_AIA_GRP_CTRL,
+		.attr	= KVM_DEV_RISCV_AIA_CTRL_INIT,
+	};
+
+	/* Setup global device attribute variables */
+	aia_mode_attr.addr = (u64)(unsigned long)&aia_mode;
+	aia_nr_ids_attr.addr = (u64)(unsigned long)&aia_nr_ids;
+	aia_nr_sources_attr.addr = (u64)(unsigned long)&aia_nr_sources;
+	aia_hart_bits_attr.addr = (u64)(unsigned long)&aia_hart_bits;
+
+	/* Do nothing if AIA device not created */
+	if (aia_fd < 0)
+		return 0;
+
+	/* Set/Get AIA device config parameters */
+	ret = ioctl(aia_fd, KVM_GET_DEVICE_ATTR, &aia_mode_attr);
+	if (ret)
+		return ret;
+	ret = ioctl(aia_fd, KVM_GET_DEVICE_ATTR, &aia_nr_ids_attr);
+	if (ret)
+		return ret;
+	aia_nr_sources = irq__get_nr_allocated_lines();
+	ret = ioctl(aia_fd, KVM_SET_DEVICE_ATTR, &aia_nr_sources_attr);
+	if (ret)
+		return ret;
+	aia_hart_bits = fls_long(kvm->nrcpus);
+	ret = ioctl(aia_fd, KVM_SET_DEVICE_ATTR, &aia_hart_bits_attr);
+	if (ret)
+		return ret;
+
+	/* Save number of HARTs for FDT generation */
+	aia_nr_harts = kvm->nrcpus;
+
+	/* Set AIA device addresses */
+	aia_addr = AIA_APLIC_ADDR(aia_nr_harts);
+	aia_addr_attr.attr = KVM_DEV_RISCV_AIA_ADDR_APLIC;
+	ret = ioctl(aia_fd, KVM_SET_DEVICE_ATTR, &aia_addr_attr);
+	if (ret)
+		return ret;
+	for (i = 0; i < kvm->nrcpus; i++) {
+		aia_addr = AIA_IMSIC_ADDR(i);
+		aia_addr_attr.attr = KVM_DEV_RISCV_AIA_ADDR_IMSIC(i);
+		ret = ioctl(aia_fd, KVM_SET_DEVICE_ATTR, &aia_addr_attr);
+		if (ret)
+			return ret;
+	}
+
+	/* Setup default IRQ routing */
+	aia__irq_routing_init(kvm);
+
+	/* Initialize the AIA device */
+	ret = ioctl(aia_fd, KVM_SET_DEVICE_ATTR, &aia_init_attr);
+	if (ret)
+		return ret;
+
+	/* Mark IRQFD as ready */
+	riscv_irqchip_irqfd_ready = true;
+
+	return 0;
+}
+late_init(aia__init);
+
+void aia__create(struct kvm *kvm)
+{
+	int err;
+	struct kvm_create_device aia_device = {
+		.type = KVM_DEV_TYPE_RISCV_AIA,
+		.flags = 0,
+	};
+
+	if (kvm->cfg.arch.ext_disabled[KVM_RISCV_ISA_EXT_SSAIA])
+		return;
+
+	err = ioctl(kvm->vm_fd, KVM_CREATE_DEVICE, &aia_device);
+	if (err)
+		return;
+	aia_fd = aia_device.fd;
+
+	riscv_irqchip = IRQCHIP_AIA;
+	riscv_irqchip_inkernel = true;
+	riscv_irqchip_trigger = NULL;
+	riscv_irqchip_generate_fdt_node = aia__generate_fdt_node;
+	riscv_irqchip_phandle = PHANDLE_AIA_APLIC;
+	riscv_irqchip_msi_phandle = PHANDLE_AIA_IMSIC;
+	riscv_irqchip_line_sensing = true;
+}
riscv/include/kvm/fdt-arch.h +7 -1
index f7548e8..d88b832 100644
@@ -1,7 +1,13 @@
 #ifndef KVM__KVM_FDT_H
 #define KVM__KVM_FDT_H
 
-enum phandles {PHANDLE_RESERVED = 0, PHANDLE_PLIC, PHANDLES_MAX};
+enum phandles {
+	PHANDLE_RESERVED = 0,
+	PHANDLE_PLIC,
+	PHANDLE_AIA_APLIC,
+	PHANDLE_AIA_IMSIC,
+	PHANDLES_MAX
+};
 
 #define PHANDLE_CPU_INTC_BASE	PHANDLES_MAX
 
riscv/include/kvm/kvm-arch.h +0 -0
index 1a8af6a..9f2159f 100644

riscv/include/kvm/kvm-arch.h extern u32 riscv_irqchip_msi_phandle; · line 104

Quoted 5 lines
riscv/include/kvm/kvm-arch.h +2 -0
@@ -100,6 +100,8 @@ extern u32 riscv_irqchip_msi_phandle;
 extern bool riscv_irqchip_line_sensing;
 extern bool riscv_irqchip_irqfd_ready;
 
+void aia__create(struct kvm *kvm);
+

nit: I'd remove the blank line between *__create functions.

Quoted 8 lines
riscv/include/kvm/kvm-arch.h +0 -0
 void plic__create(struct kvm *kvm);
 
 void pci__generate_fdt_nodes(void *fdt);
riscv/irq.c +0 -0
index e6c0939..be3e7ac 100644

riscv/irq.c void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type) · line 142

Quoted 8 lines
riscv/irq.c +3 -0
@@ -135,6 +135,9 @@ void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type)
 
 void riscv__irqchip_create(struct kvm *kvm)
 {
+	/* Try AIA in-kernel irqchip. */
+	aia__create(kvm);
+
 	/* Try PLIC irqchip */
 	plic__create(kvm);

I realize plic__create() just returns if aia__create() successful set up the irqchip, but structuring this like

ret = aia__create(kvm);
if (!ret)

    ret = plic__create(kvm);
 if (!ret)
    die(...);

Might be a little easier to for future readers of the code to grok on first read.

Either way,

Thanks,
drew

author
#12 lore raw

Re: [kvmtool PATCH v2 5/6] riscv: Use AIA in-kernel irqchip whenever KVM RISC-V supports

Quoted 85 lines · On Wed, Oct 25, 2023 at 7:09?PM Andrew Jones <[email protected]> wrote:
Quoted 69 lines · Anup Patel

The KVM RISC-V kernel module supports AIA in-kernel irqchip when underlying host has AIA support. We detect and use AIA in-kernel irqchip whenever possible otherwise we fallback to PLIC emulated in user-space.

---
 Makefile                     |   1 +
 riscv/aia.c                  | 227 +++++++++++++++++++++++++++++++++++
 riscv/include/kvm/fdt-arch.h |   8 +-
 riscv/include/kvm/kvm-arch.h |   2 +
 riscv/irq.c                  |   3 +
 5 files changed, 240 insertions(+), 1 deletion(-)
 create mode 100644 riscv/aia.c
Makefile +1 -0
index e711670..acd5ffd 100644
@@ -220,6 +220,7 @@ ifeq ($(ARCH),riscv)
      OBJS            += riscv/kvm-cpu.o
      OBJS            += riscv/pci.o
      OBJS            += riscv/plic.o
+     OBJS            += riscv/aia.o
      ifeq ($(RISCV_XLEN),32)
              CFLAGS  += -mabi=ilp32d -march=rv32gc
      endif
riscv/aia.c new file +36 -0
new file mode 100644
index 0000000..8c85b3f
@@ -0,0 +1,227 @@
+#include "kvm/devices.h"
+#include "kvm/fdt.h"
+#include "kvm/ioeventfd.h"
+#include "kvm/ioport.h"
+#include "kvm/kvm.h"
+#include "kvm/kvm-cpu.h"
+#include "kvm/irq.h"
+#include "kvm/util.h"
+
+static int aia_fd = -1;
+
+static u32 aia_mode = KVM_DEV_RISCV_AIA_MODE_EMUL;
+static struct kvm_device_attr aia_mode_attr = {
+     .group  = KVM_DEV_RISCV_AIA_GRP_CONFIG,
+     .attr   = KVM_DEV_RISCV_AIA_CONFIG_MODE,
+};
+
+static u32 aia_nr_ids = 0;
+static struct kvm_device_attr aia_nr_ids_attr = {
+     .group  = KVM_DEV_RISCV_AIA_GRP_CONFIG,
+     .attr   = KVM_DEV_RISCV_AIA_CONFIG_IDS,
+};
+
+static u32 aia_nr_sources = 0;
+static struct kvm_device_attr aia_nr_sources_attr = {
+     .group  = KVM_DEV_RISCV_AIA_GRP_CONFIG,
+     .attr   = KVM_DEV_RISCV_AIA_CONFIG_SRCS,
+};
+
+static u32 aia_hart_bits = 0;
+static struct kvm_device_attr aia_hart_bits_attr = {
+     .group  = KVM_DEV_RISCV_AIA_GRP_CONFIG,
+     .attr   = KVM_DEV_RISCV_AIA_CONFIG_HART_BITS,
+};
+
+static u32 aia_nr_harts = 0;

riscv/aia.c line 46

Quoted 10 lines
riscv/aia.c new file +10 -0
+
+#define IRQCHIP_AIA_NR                       0
+
+#define AIA_IMSIC_BASE                       RISCV_IRQCHIP
+#define AIA_IMSIC_ADDR(__hart)               \
+     (AIA_IMSIC_BASE + (__hart) * KVM_DEV_RISCV_IMSIC_SIZE)
+#define AIA_IMSIC_SIZE                       \
+     (aia_nr_harts * KVM_DEV_RISCV_IMSIC_SIZE)
+#define AIA_APLIC_ADDR(__nr_harts)   \
+     (AIA_IMSIC_BASE + (__nr_harts) * KVM_DEV_RISCV_IMSIC_SIZE)

AIA_APLIC_ADDR() probably doesn't need to take nr_harts since it's always called with aia_nr_harts. So it could just be defined as

Quoted 1 line

#define AIA_APLIC_ADDR (AIA_IMSIC_BASE + AIA_IMSIC_SIZE)

Okay, I will update.

Quoted 209 lines
Quoted 209 lines
riscv/aia.c new file +181 -0
+
+static void aia__generate_fdt_node(void *fdt, struct kvm *kvm)
+{
+     u32 i;
+     char name[64];
+     u32 reg_cells[4], *irq_cells;
+
+     irq_cells = calloc(aia_nr_harts * 2, sizeof(u32));
+     if (!irq_cells)
+             die("Failed to alloc irq_cells");
+
+     sprintf(name, "imsics@%08x", (u32)AIA_IMSIC_BASE);
+     _FDT(fdt_begin_node(fdt, name));
+     _FDT(fdt_property_string(fdt, "compatible", "riscv,imsics"));
+     reg_cells[0] = 0;
+     reg_cells[1] = cpu_to_fdt32(AIA_IMSIC_BASE);
+     reg_cells[2] = 0;
+     reg_cells[3] = cpu_to_fdt32(AIA_IMSIC_SIZE);
+     _FDT(fdt_property(fdt, "reg", reg_cells, sizeof(reg_cells)));
+     _FDT(fdt_property_cell(fdt, "#interrupt-cells", 0));
+     _FDT(fdt_property(fdt, "interrupt-controller", NULL, 0));
+     _FDT(fdt_property(fdt, "msi-controller", NULL, 0));
+     _FDT(fdt_property_cell(fdt, "riscv,num-ids", aia_nr_ids));
+     _FDT(fdt_property_cell(fdt, "phandle", PHANDLE_AIA_IMSIC));
+     for (i = 0; i < aia_nr_harts; i++) {
+             irq_cells[2*i + 0] = cpu_to_fdt32(PHANDLE_CPU_INTC_BASE + i);
+             irq_cells[2*i + 1] = cpu_to_fdt32(9);
+     }
+     _FDT(fdt_property(fdt, "interrupts-extended", irq_cells,
+                       sizeof(u32) * aia_nr_harts * 2));
+     _FDT(fdt_end_node(fdt));
+
+     free(irq_cells);
+
+     /* Skip APLIC node if we have no interrupt sources */
+     if (!aia_nr_sources)
+             return;
+
+     sprintf(name, "aplic@%08x", (u32)AIA_APLIC_ADDR(aia_nr_harts));
+     _FDT(fdt_begin_node(fdt, name));
+     _FDT(fdt_property_string(fdt, "compatible", "riscv,aplic"));
+     reg_cells[0] = 0;
+     reg_cells[1] = cpu_to_fdt32(AIA_APLIC_ADDR(aia_nr_harts));
+     reg_cells[2] = 0;
+     reg_cells[3] = cpu_to_fdt32(KVM_DEV_RISCV_APLIC_SIZE);
+     _FDT(fdt_property(fdt, "reg", reg_cells, sizeof(reg_cells)));
+     _FDT(fdt_property_cell(fdt, "#interrupt-cells", 2));
+     _FDT(fdt_property(fdt, "interrupt-controller", NULL, 0));
+     _FDT(fdt_property_cell(fdt, "riscv,num-sources", aia_nr_sources));
+     _FDT(fdt_property_cell(fdt, "phandle", PHANDLE_AIA_APLIC));
+     _FDT(fdt_property_cell(fdt, "msi-parent", PHANDLE_AIA_IMSIC));
+     _FDT(fdt_end_node(fdt));
+}
+
+static int aia__irq_routing_init(struct kvm *kvm)
+{
+     int r;
+     int irqlines = aia_nr_sources + 1;
+
+     /* Skip this if we have no interrupt sources */
+     if (!aia_nr_sources)
+             return 0;
+
+     /*
+      * This describes the default routing that the kernel uses without
+      * any routing explicitly set up via KVM_SET_GSI_ROUTING. So we
+      * don't need to commit these setting right now. The first actual
+      * user (MSI routing) will engage these mappings then.
+      */
+     for (next_gsi = 0; next_gsi < irqlines; next_gsi++) {
+             r = irq__allocate_routing_entry();
+             if (r)
+                     return r;
+
+             irq_routing->entries[irq_routing->nr++] =
+                     (struct kvm_irq_routing_entry) {
+                             .gsi = next_gsi,
+                             .type = KVM_IRQ_ROUTING_IRQCHIP,
+                             .u.irqchip.irqchip = IRQCHIP_AIA_NR,
+                             .u.irqchip.pin = next_gsi,
+             };
+     }
+
+     return 0;
+}
+
+static int aia__init(struct kvm *kvm)
+{
+     int i, ret;
+     u64 aia_addr = 0;
+     struct kvm_device_attr aia_addr_attr = {
+             .group  = KVM_DEV_RISCV_AIA_GRP_ADDR,
+             .addr   = (u64)(unsigned long)&aia_addr,
+     };
+     struct kvm_device_attr aia_init_attr = {
+             .group  = KVM_DEV_RISCV_AIA_GRP_CTRL,
+             .attr   = KVM_DEV_RISCV_AIA_CTRL_INIT,
+     };
+
+     /* Setup global device attribute variables */
+     aia_mode_attr.addr = (u64)(unsigned long)&aia_mode;
+     aia_nr_ids_attr.addr = (u64)(unsigned long)&aia_nr_ids;
+     aia_nr_sources_attr.addr = (u64)(unsigned long)&aia_nr_sources;
+     aia_hart_bits_attr.addr = (u64)(unsigned long)&aia_hart_bits;
+
+     /* Do nothing if AIA device not created */
+     if (aia_fd < 0)
+             return 0;
+
+     /* Set/Get AIA device config parameters */
+     ret = ioctl(aia_fd, KVM_GET_DEVICE_ATTR, &aia_mode_attr);
+     if (ret)
+             return ret;
+     ret = ioctl(aia_fd, KVM_GET_DEVICE_ATTR, &aia_nr_ids_attr);
+     if (ret)
+             return ret;
+     aia_nr_sources = irq__get_nr_allocated_lines();
+     ret = ioctl(aia_fd, KVM_SET_DEVICE_ATTR, &aia_nr_sources_attr);
+     if (ret)
+             return ret;
+     aia_hart_bits = fls_long(kvm->nrcpus);
+     ret = ioctl(aia_fd, KVM_SET_DEVICE_ATTR, &aia_hart_bits_attr);
+     if (ret)
+             return ret;
+
+     /* Save number of HARTs for FDT generation */
+     aia_nr_harts = kvm->nrcpus;
+
+     /* Set AIA device addresses */
+     aia_addr = AIA_APLIC_ADDR(aia_nr_harts);
+     aia_addr_attr.attr = KVM_DEV_RISCV_AIA_ADDR_APLIC;
+     ret = ioctl(aia_fd, KVM_SET_DEVICE_ATTR, &aia_addr_attr);
+     if (ret)
+             return ret;
+     for (i = 0; i < kvm->nrcpus; i++) {
+             aia_addr = AIA_IMSIC_ADDR(i);
+             aia_addr_attr.attr = KVM_DEV_RISCV_AIA_ADDR_IMSIC(i);
+             ret = ioctl(aia_fd, KVM_SET_DEVICE_ATTR, &aia_addr_attr);
+             if (ret)
+                     return ret;
+     }
+
+     /* Setup default IRQ routing */
+     aia__irq_routing_init(kvm);
+
+     /* Initialize the AIA device */
+     ret = ioctl(aia_fd, KVM_SET_DEVICE_ATTR, &aia_init_attr);
+     if (ret)
+             return ret;
+
+     /* Mark IRQFD as ready */
+     riscv_irqchip_irqfd_ready = true;
+
+     return 0;
+}
+late_init(aia__init);
+
+void aia__create(struct kvm *kvm)
+{
+     int err;
+     struct kvm_create_device aia_device = {
+             .type = KVM_DEV_TYPE_RISCV_AIA,
+             .flags = 0,
+     };
+
+     if (kvm->cfg.arch.ext_disabled[KVM_RISCV_ISA_EXT_SSAIA])
+             return;
+
+     err = ioctl(kvm->vm_fd, KVM_CREATE_DEVICE, &aia_device);
+     if (err)
+             return;
+     aia_fd = aia_device.fd;
+
+     riscv_irqchip = IRQCHIP_AIA;
+     riscv_irqchip_inkernel = true;
+     riscv_irqchip_trigger = NULL;
+     riscv_irqchip_generate_fdt_node = aia__generate_fdt_node;
+     riscv_irqchip_phandle = PHANDLE_AIA_APLIC;
+     riscv_irqchip_msi_phandle = PHANDLE_AIA_IMSIC;
+     riscv_irqchip_line_sensing = true;
+}
riscv/include/kvm/fdt-arch.h +7 -1
index f7548e8..d88b832 100644
@@ -1,7 +1,13 @@
 #ifndef KVM__KVM_FDT_H
 #define KVM__KVM_FDT_H
 
-enum phandles {PHANDLE_RESERVED = 0, PHANDLE_PLIC, PHANDLES_MAX};
+enum phandles {
+     PHANDLE_RESERVED = 0,
+     PHANDLE_PLIC,
+     PHANDLE_AIA_APLIC,
+     PHANDLE_AIA_IMSIC,
+     PHANDLES_MAX
+};
 
 #define PHANDLE_CPU_INTC_BASE        PHANDLES_MAX
 
riscv/include/kvm/kvm-arch.h +0 -0
index 1a8af6a..9f2159f 100644
@@ -100,6 +100,8 @@ extern u32 riscv_irqchip_msi_phandle;
 extern bool riscv_irqchip_line_sensing;
 extern bool riscv_irqchip_irqfd_ready;
 
Quoted 4 lines

riscv/include/kvm/kvm-arch.h extern u32 riscv_irqchip_msi_phandle; · line 104

Quoted 2 lines
riscv/include/kvm/kvm-arch.h +2 -0
+void aia__create(struct kvm *kvm);
+

nit: I'd remove the blank line between *__create functions.

Okay, I will update.

Quoted 21 lines
Quoted 9 lines
riscv/include/kvm/kvm-arch.h +0 -0
 void plic__create(struct kvm *kvm);
 
 void pci__generate_fdt_nodes(void *fdt);
riscv/irq.c +0 -0
index e6c0939..be3e7ac 100644

riscv/irq.c void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type) · line 142

Quoted 8 lines
riscv/irq.c +3 -0
@@ -135,6 +135,9 @@ void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type)
 
 void riscv__irqchip_create(struct kvm *kvm)
 {
+     /* Try AIA in-kernel irqchip. */
+     aia__create(kvm);
+
      /* Try PLIC irqchip */
      plic__create(kvm);

I realize plic__create() just returns if aia__create() successful set up the irqchip, but structuring this like

Quoted 8 lines

ret = aia__create(kvm);
if (!ret)

    ret = plic__create(kvm);
 if (!ret)
    die(...);

Might be a little easier to for future readers of the code to grok on first read.

Okay, I will update it so that it is obvious to readers that __create() functions are tried one after another until one suceeds.

Quoted 8 lines

Either way,

Thanks,
drew

Regards,
Anup

author
#13 lore raw

PATCH v2 6/6 riscv: Fix guest/init linkage for multilib toolchain

In reply to Anup Patel

For RISC-V multilib toolchains, we must specify -mabi and -march options when linking guest/init.

Fixes 2e99678314c2 ("riscv: Initial skeletal support")
---
 Makefile | 2 ++
 1 file changed, 2 insertions(+)
Makefile +2 -0
index acd5ffd..d84dc8e 100644
@@ -223,9 +223,11 @@ ifeq ($(ARCH),riscv)
 	OBJS		+= riscv/aia.o
 	ifeq ($(RISCV_XLEN),32)
 		CFLAGS	+= -mabi=ilp32d -march=rv32gc
+		GUEST_INIT_FLAGS += -mabi=ilp32d -march=rv32gc
 	endif
 	ifeq ($(RISCV_XLEN),64)
 		CFLAGS	+= -mabi=lp64d -march=rv64gc
+		GUEST_INIT_FLAGS += -mabi=lp64d -march=rv64gc
 	endif
 
 	ARCH_WANT_LIBFDT := y
Signature · 1 line
2.34.1
Reviewed-by on v2 6/6
#14 lore raw

Re: [kvmtool PATCH v2 6/6] riscv: Fix guest/init linkage for multilib toolchain

Quoted 28 lines · Anup Patel

For RISC-V multilib toolchains, we must specify -mabi and -march options when linking guest/init.

Fixes 2e99678314c2 ("riscv: Initial skeletal support")
---
 Makefile | 2 ++
 1 file changed, 2 insertions(+)
Makefile +2 -0
index acd5ffd..d84dc8e 100644
@@ -223,9 +223,11 @@ ifeq ($(ARCH),riscv)
 	OBJS		+= riscv/aia.o
 	ifeq ($(RISCV_XLEN),32)
 		CFLAGS	+= -mabi=ilp32d -march=rv32gc
+		GUEST_INIT_FLAGS += -mabi=ilp32d -march=rv32gc
 	endif
 	ifeq ($(RISCV_XLEN),64)
 		CFLAGS	+= -mabi=lp64d -march=rv64gc
+		GUEST_INIT_FLAGS += -mabi=lp64d -march=rv64gc
 	endif
 
 	ARCH_WANT_LIBFDT := y
Signature · 1 line
2.34.1

Looks good and like something that could be posted independently of this series.

Thanks,
drew

author
#15 lore raw
In reply to Anup Patel

Hi Will,

Quoted 15 lines · On Mon, Sep 18, 2023 at 6:27?PM Anup Patel <[email protected]> wrote:

The latest KVM in Linux-6.5 has support for:
1) Svnapot ISA extension support
2) AIA in-kernel irqchip support

This series adds corresponding changes in KVMTOOL to use the above mentioned features for Guest/VM.

These patches can also be found in the riscv_aia_v2 branch at:
https://github.com/avpatel/kvmtool.git

Changes since v1:
- Rebased on commit 9cb1b46cb765972326a46bdba867d441a842af56
- Updated PATCH1 to sync header with released Linux-6.5

Quoted 7 lines

Anup Patel (6):
Sync-up header with Linux-6.5 for KVM RISC-V
riscv: Add Svnapot extension support
riscv: Make irqchip support pluggable
riscv: Add IRQFD support for in-kernel AIA irqchip
riscv: Use AIA in-kernel irqchip whenever KVM RISC-V supports
riscv: Fix guest/init linkage for multilib toolchain

Friendly ping ?

Quoted 16 lines
 Makefile                            |   3 +
 include/linux/kvm.h                 |   6 +-
 riscv/aia.c                         | 227 ++++++++++++++++++++++++++++
 riscv/fdt.c                         |  15 +-
 riscv/include/asm/kvm.h             |  81 ++++++++++
 riscv/include/kvm/fdt-arch.h        |   8 +-
 riscv/include/kvm/kvm-arch.h        |  38 ++++-
 riscv/include/kvm/kvm-config-arch.h |   3 +
 riscv/irq.c                         | 138 ++++++++++++++++-
 riscv/kvm.c                         |   2 +
 riscv/pci.c                         |  32 ++--
 riscv/plic.c                        |  61 ++++----
 12 files changed, 563 insertions(+), 51 deletions(-)
 create mode 100644 riscv/aia.c
Quoted 3 lines

--
2.34.1

Regards,
Anup

#16 lore raw
Quoted 24 lines · Anup Patel
Quoted 15 lines · On Mon, Sep 18, 2023 at 6:27?PM Anup Patel <[email protected]> wrote:

The latest KVM in Linux-6.5 has support for:
1) Svnapot ISA extension support
2) AIA in-kernel irqchip support

This series adds corresponding changes in KVMTOOL to use the above mentioned features for Guest/VM.

These patches can also be found in the riscv_aia_v2 branch at:
https://github.com/avpatel/kvmtool.git

Changes since v1:
- Rebased on commit 9cb1b46cb765972326a46bdba867d441a842af56
- Updated PATCH1 to sync header with released Linux-6.5

Quoted 7 lines

Anup Patel (6):
Sync-up header with Linux-6.5 for KVM RISC-V
riscv: Add Svnapot extension support
riscv: Make irqchip support pluggable
riscv: Add IRQFD support for in-kernel AIA irqchip
riscv: Use AIA in-kernel irqchip whenever KVM RISC-V supports
riscv: Fix guest/init linkage for multilib toolchain

Quoted 1 line

Friendly ping ?

There are a bunch of open review comments from Drew that need to be addressed in a subsequent version.

Will

author
#17 lore raw
Quoted 26 lines · On Tue, Nov 7, 2023 at 4:41?PM Will Deacon <[email protected]> wrote:
Quoted 24 lines · Anup Patel
Quoted 22 lines · On Mon, Sep 18, 2023 at 6:27?PM Anup Patel <[email protected]> wrote:

The latest KVM in Linux-6.5 has support for:
1) Svnapot ISA extension support
2) AIA in-kernel irqchip support

This series adds corresponding changes in KVMTOOL to use the above mentioned features for Guest/VM.

These patches can also be found in the riscv_aia_v2 branch at:
https://github.com/avpatel/kvmtool.git

Changes since v1:
- Rebased on commit 9cb1b46cb765972326a46bdba867d441a842af56
- Updated PATCH1 to sync header with released Linux-6.5

Anup Patel (6):
Sync-up header with Linux-6.5 for KVM RISC-V
riscv: Add Svnapot extension support
riscv: Make irqchip support pluggable
riscv: Add IRQFD support for in-kernel AIA irqchip
riscv: Use AIA in-kernel irqchip whenever KVM RISC-V supports
riscv: Fix guest/init linkage for multilib toolchain

Quoted 4 lines
Quoted 1 line

Friendly ping ?

There are a bunch of open review comments from Drew that need to be addressed in a subsequent version.

I have sent-out v3 with Drew's comments addressed.

Thanks,
Anup