powerpc: Cleanup and convert to MSI parent domain
Subsystems (2)
v2 cover powerpc: Cleanup and convert to MSI parent domain
The initial implementation of PCI/MSI interrupt domains in the hierarchical interrupt domain model used a shortcut by providing a global PCI/MSI domain.
This works because the PCI/MSI[X] hardware is standardized and uniform, but it violates the basic design principle of hierarchical interrupt domains:
Each hardware block involved in the interrupt delivery chain should have a separate interrupt domain.
For PCI/MSI[X], the interrupt controller is per PCI device and not a global made-up entity.
Unsurprisingly, the shortcut turned out to have downsides as it does not allow dynamic allocation of interrupt vectors after initialization and it prevents supporting IMS on PCI. For further details, see:
https://lore.kernel.org/lkml/[email protected]/
The solution is implementing per device MSI domains, this means the entities which provide global PCI/MSI domain so far have to implement MSI parent domain functionality instead.
This series:
- Untangle XIVE driver from Powernv and Pseries drivers
- Convert the Powernv and Pseries drivers to implement MSI parent domain functionality
v2: Fix up boot issue with NVMe
Nam Cao (3):
powerpc/xive: Untangle xive from child interrupt controller drivers powerpc/powernv/pci: Switch to use msi_create_parent_irq_domain() powerpc/pseries/msi: Switch to msi_create_parent_irq_domain()
arch/powerpc/include/asm/pci-bridge.h | 2 - arch/powerpc/include/asm/xive.h | 1 - arch/powerpc/platforms/powernv/Kconfig | 1 + arch/powerpc/platforms/powernv/pci-ioda.c | 96 ++++++---------- arch/powerpc/platforms/pseries/Kconfig | 1 + arch/powerpc/platforms/pseries/msi.c | 132 ++++++++-------------- arch/powerpc/sysdev/xive/common.c | 63 +++++------ 7 files changed, 117 insertions(+), 179 deletions(-)
Signature · 1 line
2.39.5
PATCH v2 1/3 powerpc/xive: Untangle xive from child interrupt controller drivers
xive-specific data is stored in handler_data. This creates a mess, as xive has to rely on child interrupt controller drivers to clean up this data, as was done by 9a014f45688 ("powerpc/pseries/pci: Add a msi_free() handler to clear XIVE data").
Instead, store xive-specific data in chip_data and untangle the child drivers.
---
v2: no change
--- arch/powerpc/include/asm/xive.h | 1 - arch/powerpc/platforms/powernv/pci-ioda.c | 21 +------- arch/powerpc/platforms/pseries/msi.c | 18 +------ arch/powerpc/sysdev/xive/common.c | 63 +++++++++++------------ 4 files changed, 33 insertions(+), 70 deletions(-)
arch/powerpc/include/asm/xive.h +0 -1
@@ -111,7 +111,6 @@ void xive_native_free_vp_block(u32 vp_base);int xive_native_populate_irq_data (u32 hw_irq,struct xive_irq_data *data);void xive_cleanup_irq_data (struct xive_irq_data *xd); -void xive_irq_free_data (unsigned int virq);void xive_native_free_irq (u32 irq);int xive_native_configure_irq (u32 hw_irq,u32 target,u8 prio,u32 sw_irq);
arch/powerpc/platforms/powernv/pci-ioda.c +1 -20
@@ -37,7 +37,6 @@#include <asm/firmware.h> #include <asm/pnv-pci.h> #include <asm/mmzone.h> -#include <asm/xive.h> #include "powernv.h" #include "pci.h"
@@ -1707,23 +1706,6 @@ static int __pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev,return 0 ; } -/* -* The msi_free() op is called before irq_domain_free_irqs_top() when -* the handler data is still available. Use that to clear the XIVE -* controller. -*/ -static void pnv_msi_ops_msi_free (struct irq_domain *domain, -struct msi_domain_info *info, -unsigned int irq) -{ -if (xive_enabled ()) -xive_irq_free_data (irq); -} - -static struct msi_domain_ops pnv_pci_msi_domain_ops = { - .msi_free = pnv_msi_ops_msi_free, -}; -static void pnv_msi_shutdown (struct irq_data *d) { d = d->parent_data;
@@ -1754,7 +1736,6 @@ static struct irq_chip pnv_pci_msi_irq_chip = {static struct msi_domain_info pnv_msi_domain_info = { .flags = (MSI_FLAG_USE_DEF_DOM_OPS |MSI_FLAG_USE_DEF_CHIP_OPS |MSI_FLAG_MULTI_PCI_MSI |MSI_FLAG_PCI_MSIX ), - .ops = &pnv_pci_msi_domain_ops, .chip = &pnv_pci_msi_irq_chip, };
@@ -1870,7 +1851,7 @@ static void pnv_irq_domain_free(struct irq_domain *domain, unsigned int virq, virq, d->hwirq, nr_irqs);msi_bitmap_free_hwirqs (&phb->msi_bmp, d->hwirq, nr_irqs); -/* XIVE domain is cleared through ->msi_free() */ +irq_domain_free_irqs_parent (domain, virq, nr_irqs); }static const struct irq_domain_ops pnv_irq_domain_ops = {
arch/powerpc/platforms/pseries/msi.c +1 -17
@@ -15,7 +15,6 @@#include <asm/hw_irq.h> #include <asm/ppc-pci.h> #include <asm/machdep.h> -#include <asm/xive.h> #include "pseries.h"
@@ -436,19 +435,6 @@ static int pseries_msi_ops_prepare(struct irq_domain *domain, struct device *devreturn rtas_prepare_msi_irqs (pdev, nvec, type, arg); } -/* -* ->msi_free() is called before irq_domain_free_irqs_top() when the -* handler data is still available. Use that to clear the XIVE -* controller data. -*/ -static void pseries_msi_ops_msi_free (struct irq_domain *domain, -struct msi_domain_info *info, -unsigned int irq) -{ -if (xive_enabled ()) -xive_irq_free_data (irq); -} -/* * RTAS can not disable one MSI at a time. It's all or nothing. Do it * at the end after all IRQs have been freed.
@@ -463,7 +449,6 @@ static void pseries_msi_post_free(struct irq_domain *domain, struct device *dev)static struct msi_domain_ops pseries_pci_msi_domain_ops = { .msi_prepare = pseries_msi_ops_prepare, - .msi_free = pseries_msi_ops_msi_free, .msi_post_free = pseries_msi_post_free, };
@@ -604,8 +589,7 @@ static void pseries_irq_domain_free(struct irq_domain *domain, unsigned int virqstruct pci_controller *phb =irq_data_get_irq_chip_data (d);pr_debug ("%s bridge %pOF %d #%d\n" , __func__, phb->dn, virq, nr_irqs); - -/* XIVE domain data is cleared through ->msi_free() */ +irq_domain_free_irqs_parent (domain, virq, nr_irqs); }static const struct irq_domain_ops pseries_irq_domain_ops = {
arch/powerpc/sysdev/xive/common.c +31 -32
206 lines not on this page · raw message
Signature · 1 line
2.39.5
Ritesh Harjani (IBM)
reviewerRe: [PATCH v2 1/3] powerpc/xive: Untangle xive from child interrupt controller drivers
Quoted 9 lines · Nam Cao
xive-specific data is stored in
handler_data. This creates a mess, as xive has to rely on child interrupt controller drivers to clean up this data, as was done by9a014f45688("powerpc/pseries/pci: Add amsi_free()handler to clear XIVE data").Instead, store xive-specific data in
chip_dataand untangle the child drivers.Signed-off-by Nam Cao <[email protected]>
Quoted 8 lines
---
v2: no change--- arch/powerpc/include/asm/xive.h | 1 - arch/powerpc/platforms/powernv/pci-ioda.c | 21 +------- arch/powerpc/platforms/pseries/msi.c | 18 +------ arch/powerpc/sysdev/xive/common.c | 63 +++++++++++------------ 4 files changed, 33 insertions(+), 70 deletions(-)
Hi Nam,
I am facing kernel crash on host when trying to run kvm pseries guest on powernv host. Looking it a bit more closely, I see that we are missing conversion of xxx_irq_handler_data()) to xxx_irq_chip_data() at few other places, including in powerpc KVM code.
<Crash signature>
BUG: Kernel NULL pointer dereference on read at 0x00000010 Faulting instruction address: 0xc0000000001c0704 Oops: Kernel access of bad area, sig: 11 [#1] LE PAGE_SIZE=64K MMU=Radix SMP NR_CPUS=2048 NUMA PowerNV CPU: 103 UID: 0 PID: 2742 Comm: qemu-system-ppc Not tainted 6.17.0-01737-g50c19e20ed2e #1 NONE <...> NIP: c0000000001c0704 LR: c0000000001c06f8 CTR: 0000000000000000 REGS: c0000000627476a0 TRAP: 0300 Not tainted (6.17.0-01737-g50c19e20ed2e) MSR: 9000000000009033 <SF,HV,EE,ME,IR,DR,RI,LE> CR: 88044488 XER: 00000036 CFAR: c0000000002c20d8 DAR: 0000000000000010 DSISR: 40000000 IRQMASK: 0 <...> NIP [c0000000001c0704] kvmppc_xive_attach_escalation+0x174/0x240 LR [c0000000001c06f8] kvmppc_xive_attach_escalation+0x168/0x240 Call Trace: kvmppc_xive_attach_escalation+0x168/0x240 (unreliable) kvmppc_xive_connect_vcpu+0x2a0/0x4c0 kvm_arch_vcpu_ioctl+0x354/0x470 kvm_vcpu_ioctl+0x488/0x9a0 sys_ioctl+0x4ec/0x1030 system_call_exception+0x104/0x2b0 system_call_vectored_common+0x15c/0x2ec
Here is the diff which fixed this..
arch/powerpc/kvm/book3s_xive.c +4 -4
@@ -917,7 +917,7 @@ int kvmppc_xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio,*/ if (single_escalation) {struct irq_data *d =irq_get_irq_data (xc->esc_virq[prio]); -struct xive_irq_data *xd =irq_data_get_irq_handler_data (d); +struct xive_irq_data *xd =irq_data_get_irq_chip_data (d);xive_vm_esb_load (xd,XIVE_ESB_SET_PQ_01 ); vcpu->arch.xive_esc_raddr = xd->eoi_page;
@@ -1612,7 +1612,7 @@ int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq,/* Grab info about irq */ state->pt_number = hw_irq; - state->pt_data =irq_data_get_irq_handler_data (host_data); + state->pt_data =irq_data_get_irq_chip_data (host_data);/* * Configure the IRQ to match the existing configuration of
@@ -1788,7 +1788,7 @@ void kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu *vcpu)void xive_cleanup_single_escalation (struct kvm_vcpu *vcpu,int irq) {struct irq_data *d =irq_get_irq_data (irq); -struct xive_irq_data *xd =irq_data_get_irq_handler_data (d); +struct xive_irq_data *xd =irq_data_get_irq_chip_data (d);/* * This slightly odd sequence gives the right result
@@ -2829,7 +2829,7 @@ int kvmppc_xive_debug_show_queues(struct seq_file *m, struct kvm_vcpu *vcpu)if (xc->esc_virq[i]) {struct irq_data *d =irq_get_irq_data (xc->esc_virq[i]);struct xive_irq_data *xd = -irq_data_get_irq_handler_data (d); +irq_data_get_irq_chip_data (d);u64 pq =xive_vm_esb_load (xd,XIVE_ESB_GET ); seq_printf(m, " ESC %d %c%c EOI @%llx",
... However grepping for "handler_data" in arch/powerpc I see there is atleast one more place where we may still need the fix.. There are few more places which grep returned - but I am not sure if they all really need the fix. But I guess VAS should be fixed i.e :
arch/powerpc/platforms/powernv/vas.c: xd = irq_get_handler_data(vinst->virq);
Would you like to submit an official patch for converting these other places too?
Thanks!
-ritesh
Re: [PATCH v2 1/3] powerpc/xive: Untangle xive from child interrupt controller drivers
Hi Ritesh,
Quoted 4 lines · Ritesh Harjani (IBM)
I am facing kernel crash on host when trying to run kvm pseries guest on powernv host. Looking it a bit more closely, I see that we are missing conversion of
xxx_irq_handler_data()) toxxx_irq_chip_data()at few other places, including in powerpc KVM code.
[snip]
Quoted 1 line
Here is the diff which fixed this..
[snip]
Quoted 8 lines
... However grepping for "
handler_data" inarch/powerpcI see there is atleast one more place where we may still need the fix.. There are few more places which grep returned - but I am not sure if they all really need the fix. But I guess VAS should be fixed i.e :
arch/powerpc/platforms/powernv/vas.c: xd =irq_get_handler_data(vinst->virq);Would you like to submit an official patch for converting these other places too?
Thanks for the report. I didn't expect struct xive_irq_data to be used in multiple files while making that patch, sorry about that!
Something like your patch should do the job. However, my gut feeling is that multiple files shouldn't share a single irq struct this way. Let me stare at it...
Nam
Re: [PATCH v2 1/3] powerpc/xive: Untangle xive from child interrupt controller drivers
Quoted 17 lines · Nam Cao
Hi Ritesh,
Quoted 4 lines · Ritesh Harjani (IBM)
I am facing kernel crash on host when trying to run kvm pseries guest on powernv host. Looking it a bit more closely, I see that we are missing conversion of
xxx_irq_handler_data()) toxxx_irq_chip_data()at few other places, including in powerpc KVM code.[snip]
Quoted 1 line
Here is the diff which fixed this..
[snip]
Quoted 7 lines
... However grepping for "
handler_data" inarch/powerpcI see there is atleast one more place where we may still need the fix.. There are few more places which grep returned - but I am not sure if they all really need the fix. But I guess VAS should be fixed i.e :
arch/powerpc/platforms/powernv/vas.c: xd =irq_get_handler_data(vinst->virq);
Quoted 8 lines
Quoted 1 line
Would you like to submit an official patch for converting these other places too?
Thanks for the report. I didn't expect struct
xive_irq_datato be used in multiple files while making that patch, sorry about that!Something like your patch should do the job. However, my gut feeling is that multiple files shouldn't share a single irq struct this way. Let me stare at it...
I *think* we can do a cleanup. But I don't think it would be trivial enough for 6.18. Let's do as you suggested for now to get KVM functional again.
Nam
PATCH v2 2/3 powerpc/powernv/pci: Switch to use msi_create_parent_irq_domain()
In reply to Nam CaoMove away from the legacy MSI domain setup, switch to use msi_create_parent_irq_domain().
---
v2: no change
--- arch/powerpc/platforms/powernv/Kconfig | 1 + arch/powerpc/platforms/powernv/pci-ioda.c | 75 ++++++++++------------- 2 files changed, 35 insertions(+), 41 deletions(-)
arch/powerpc/platforms/powernv/Kconfig +1 -0
@@ -9,6 +9,7 @@ config PPC_POWERNV select PPC_P7_NAP select FORCE_PCI select PCI_MSI + select IRQ_MSI_LIB select EPAPR_BOOT select PPC_INDIRECT_PIO select PPC_UDBG_16550
arch/powerpc/platforms/powernv/pci-ioda.c +34 -41
107 lines not on this page · raw message
Signature · 1 line
2.39.5
PATCH v2 3/3 powerpc/pseries/msi: Switch to msi_create_parent_irq_domain()
In reply to Nam CaoMove away from the legacy MSI domain setup, switch to use msi_create_parent_irq_domain().
---
v2: Use msi_teardown() instead of msi_post_free() to fix boot issue with
NVMe
--- arch/powerpc/include/asm/pci-bridge.h | 2 - arch/powerpc/platforms/pseries/Kconfig | 1 + arch/powerpc/platforms/pseries/msi.c | 114 +++++++++++-------------- 3 files changed, 49 insertions(+), 68 deletions(-)
arch/powerpc/include/asm/pci-bridge.h +0 -2
@@ -133,8 +133,6 @@ struct pci_controller {/* IRQ domain hierarchy */ struct irq_domain *dev_domain; -struct irq_domain *msi_domain; -struct fwnode_handle *fwnode;/* iommu_ops support */ struct iommu_device iommu;
arch/powerpc/platforms/pseries/Kconfig +1 -0
@@ -7,6 +7,7 @@ config PPC_PSERIES select OF_DYNAMIC select FORCE_PCI select PCI_MSI + select IRQ_MSI_LIB select GENERIC_ALLOCATOR select PPC_XICS select PPC_XIVE_SPAPR
arch/powerpc/platforms/pseries/msi.c +48 -66
174 lines not on this page · raw message
Signature · 1 line
2.39.5
Quoted 22 lines · Nam Cao
The initial implementation of PCI/MSI interrupt domains in the hierarchical interrupt domain model used a shortcut by providing a global PCI/MSI domain.
This works because the PCI/MSI[X] hardware is standardized and uniform, but it violates the basic design principle of hierarchical interrupt domains:
Each hardware block involved in the interrupt delivery chain should have a separate interrupt domain.For PCI/MSI[X], the interrupt controller is per PCI device and not a global made-up entity.
Unsurprisingly, the shortcut turned out to have downsides as it does not allow dynamic allocation of interrupt vectors after initialization and it prevents supporting IMS on PCI. For further details, see:
https://lore.kernel.org/lkml/[email protected]/
The solution is implementing per device MSI domains, this means the entities which provide global PCI/MSI domain so far have to implement MSI parent domain functionality instead.
Quoted 6 lines
This series:
- Untangle XIVE driver from Powernv and Pseries drivers
- Convert the Powernv and Pseries drivers to implement MSI parent domain functionality
Polite reminder to the PPC folks. Can we please get this moving so we can finally cleanup the pci_msi_create_irq_domain() leftovers?
Thanks,
tglx
Madhavan Srinivasan
maintainerQuoted 26 lines · Thomas Gleixner
Quoted 24 lines · Nam Cao
The initial implementation of PCI/MSI interrupt domains in the hierarchical interrupt domain model used a shortcut by providing a global PCI/MSI domain.
This works because the PCI/MSI[X] hardware is standardized and uniform, but it violates the basic design principle of hierarchical interrupt domains:
Each hardware block involved in the interrupt delivery chain should have a separate interrupt domain.For PCI/MSI[X], the interrupt controller is per PCI device and not a global made-up entity.
Unsurprisingly, the shortcut turned out to have downsides as it does not allow dynamic allocation of interrupt vectors after initialization and it prevents supporting IMS on PCI. For further details, see:
https://lore.kernel.org/lkml/[email protected]/
The solution is implementing per device MSI domains, this means the entities which provide global PCI/MSI domain so far have to implement MSI parent domain functionality instead.
This series:
Quoted 7 lines
Quoted 4 lines
- Untangle XIVE driver from Powernv and Pseries drivers
- Convert the Powernv and Pseries drivers to implement MSI parent domain functionality
Polite reminder to the PPC folks. Can we please get this moving so we can finally cleanup the
pci_msi_create_irq_domain()leftovers?
Yes, sorry for the delay,
Will add this to my next after the test
Maddy
Quoted 4 lines
Thanks,
tglx
Quoted 10 lines · Nam Cao
The initial implementation of PCI/MSI interrupt domains in the hierarchical interrupt domain model used a shortcut by providing a global PCI/MSI domain.
This works because the PCI/MSI[X] hardware is standardized and uniform, but it violates the basic design principle of hierarchical interrupt domains:
Each hardware block involved in the interrupt delivery chain should have a separate interrupt domain.[...]
Applied to powerpc/next.
[1/3] powerpc/xive: Untangle xive from child interrupt controller drivers
https://git.kernel.org/powerpc/c/cc0cc23babc979e399f34f53e4bccf702a389558 [2/3] powerpc/powernv/pci: Switch to use msi_create_parent_irq_domain() https://git.kernel.org/powerpc/c/f0ac60e6e311062f1a452d93376055787db4b070 [3/3] powerpc/pseries/msi: Switch to msi_create_parent_irq_domain() https://git.kernel.org/powerpc/c/daaa574aba6f9c683408b58a7ab2dc775ece2f98
Thanks