PCI: quirks: Add warm reset quirk for IBM Spyre accelerator
PATCH v2 1/1 PCI: quirks: Add warm reset quirk for IBM Spyre accelerator
From: Shivaprasad G Bhat <[email protected]>
The IBM Spyre accelerator (PCI ID 1014:06a7) requires a dedicated warm reset method for reliable EEH recovery on pSeries/ppc64 platforms.
FLR alone is insufficient because it does not re-initialize firmware state machine for the current version of the card. The existing API pci_set_pcie_reset_state() is used to drive the platform-level PERST# using the EEH interfaces by adding a device specific quirk.
The reset function is placed in arch/powerpc/platforms/pseries/pci.c where the EEH infrastructure is already present, declared in pseries.h, and registered in the generic pci_dev_reset_methods[] table in drivers/pci/quirks.c under a CONFIG_PPC_PSERIES guard.
This reset method is only registered when CONFIG_PPC_PSERIES is enabled, so it is not available on other platforms.
Reset sequence:
- Assert PERST# via pcie_warm_reset, hold 250 ms
- Deassert PERST# via pcie_deassert_reset
- Wait 250 ms for PCIe link retrain
---
Changelog
v1: https://lore.kernel.org/linux-pci/[email protected]/
Reworded the commit message to highlight the warm reset requirement being a need of the current card and not a platform deficiency.
arch/powerpc/platforms/pseries/pci.c +36 -0
@@ -11,6 +11,7 @@#include <linux/kernel.h> #include <linux/pci.h> #include <linux/string.h> +#include <linux/delay.h> #include <asm/eeh.h> #include <asm/pci-bridge.h>
@@ -236,6 +237,41 @@ static void fixup_winbond_82c105(struct pci_dev* dev)DECLARE_PCI_FIXUP_HEADER (PCI_VENDOR_ID_WINBOND ,PCI_DEVICE_ID_WINBOND_82C105 , fixup_winbond_82c105); +/* +* Reset Spyre adapter using pci_set_pcie_reset_state() +* This is specifically for PPC platforms where EEH (Enhanced Error Handling) +* requires this reset method for proper device recovery. +*/ +int reset_spyre (struct pci_dev *dev,bool probe) +{ +int ret; + +if (probe) +return 0 ; + +/* Assert warm reset */ + ret =pci_set_pcie_reset_state (dev, pcie_warm_reset); +if (ret) { +pci_err (dev,"Failed to assert reset: %d\n" , ret); +return ret; + } + +/* Wait for reset to take effect */ +msleep (250 ); + +/* Deassert reset */ + ret =pci_set_pcie_reset_state (dev, pcie_deassert_reset); +if (ret) { +pci_err (dev,"Failed to deassert reset: %d\n" , ret); +return ret; + } + +/* Wait for device to recover */ +msleep (250 ); + +return 0 ; +} +static enum pci_bus_speed prop_to_pci_speed (u32 prop) {switch (prop) {
arch/powerpc/platforms/pseries/pseries.h +4 -0
@@ -92,6 +92,10 @@ extern struct pci_controller_ops pseries_pci_controller_ops;int pseries_msi_allocate_domains (struct pci_controller *phb);void pseries_msi_free_domains (struct pci_controller *phb); +/* PCI device reset method for Spyre adapter */ +struct pci_dev ; +int reset_spyre (struct pci_dev *dev,bool probe); +extern int CMO_PrPSP;extern int CMO_SecPSP;extern unsigned long CMO_PageSize;
drivers/pci/quirks.c +9 -0
@@ -4237,6 +4237,11 @@ static int reset_hinic_vf_dev(struct pci_dev *pdev, bool probe)return 0 ; } +#ifdef CONFIG_PPC_PSERIES +/* Defined in arch/powerpc/platforms/pseries/pci.c */ +int reset_spyre (struct pci_dev *dev,bool probe); +#endif +static const struct pci_dev_reset_methods pci_dev_reset_methods[] = { {PCI_VENDOR_ID_INTEL ,PCI_DEVICE_ID_INTEL_82599_SFP_VF , reset_intel_82599_sfp_virtfn },
@@ -4252,6 +4257,10 @@ static const struct pci_dev_reset_methods pci_dev_reset_methods[] = { reset_chelsio_generic_dev }, { PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HINIC_VF, reset_hinic_vf_dev }, +#ifdef CONFIG_PPC_PSERIES + { PCI_VENDOR_ID_IBM, 0x06a7, + reset_spyre}, +#endif { 0 } };
Quoted 60 lines · Avinash Roy
From: Shivaprasad G Bhat <[email protected]>
The IBM Spyre accelerator (PCI ID 1014:06a7) requires a dedicated warm reset method for reliable EEH recovery on pSeries/ppc64 platforms.
FLR alone is insufficient because it does not re-initialize firmware state machine for the current version of the card. The existing API
pci_set_pcie_reset_state()is used to drive the platform-level PERST# using the EEH interfaces by adding a device specific quirk.The reset function is placed in
arch/powerpc/platforms/pseries/pci.cwhere the EEH infrastructure is already present, declared inpseries.h, and registered in the generic pci_dev_reset_methods[] table indrivers/pci/quirks.cunder aCONFIG_PPC_PSERIESguard.This reset method is only registered when
CONFIG_PPC_PSERIESis enabled, so it is not available on other platforms.Reset sequence:
- Assert PERST# viapcie_warm_reset, hold 250 ms
- Deassert PERST# viapcie_deassert_reset
- Wait 250 ms for PCIe link retrainSigned-off-by Avinash Roy <[email protected]>Signed-off-by Shivaprasad G Bhat <[email protected]>---
Changelog
v1: https://lore.kernel.org/linux-pci/[email protected]/Reworded the commit message to highlight the warm reset requirement being a need of the current card and not a platform deficiency.
arch/powerpc/platforms/pseries/pci.c +12
-0@@ -11,6 +11,7 @@#include <linux/kernel.h> #include <linux/pci.h> #include <linux/string.h> +#include <linux/delay.h> #include <asm/eeh.h> #include <asm/pci-bridge.h> @@ -236,6 +237,41 @@ static void fixup_winbond_82c105(struct pci_dev* dev)DECLARE_PCI_FIXUP_HEADER (PCI_VENDOR_ID_WINBOND ,PCI_DEVICE_ID_WINBOND_82C105 , fixup_winbond_82c105); +/* +* Reset Spyre adapter using pci_set_pcie_reset_state() +* This is specifically for PPC platforms where EEH (Enhanced Error Handling) +* requires this reset method for proper device recovery. +*/ +int reset_spyre (struct pci_dev *dev,bool probe) +{ +int ret; + +if (probe) +return 0 ;
arch/powerpc/platforms/pseries/pci.c static void fixup_winbond_82c105(struct pci_dev* dev) · line 260
Quoted 10 lines
arch/powerpc/platforms/pseries/pci.c +10
-0+ +/* Assert warm reset */ + ret =pci_set_pcie_reset_state (dev, pcie_warm_reset); +if (ret) { +pci_err (dev,"Failed to assert reset: %d\n" , ret); +return ret; + } + +/* Wait for reset to take effect */ +msleep (250 );
How did you arriver on this number ? Is this time defined by hardware/firmware specs for reset ? Also, Can you #define the value 250 and use macro instead of constant numbmer.
arch/powerpc/platforms/pseries/pci.c static void fixup_winbond_82c105(struct pci_dev* dev) · line 270
Quoted 10 lines
arch/powerpc/platforms/pseries/pci.c +10
-0+ +/* Deassert reset */ + ret =pci_set_pcie_reset_state (dev, pcie_deassert_reset); +if (ret) { +pci_err (dev,"Failed to deassert reset: %d\n" , ret); +return ret; + } + +/* Wait for device to recover */ +msleep (250 );
same here.
arch/powerpc/platforms/pseries/pci.c static void fixup_winbond_82c105(struct pci_dev* dev) · line 258
Quoted 4 lines
arch/powerpc/platforms/pseries/pci.c +4
-0+ +return 0 ; +} +
Thanks,
-Mahesh.
Quoted 66 lines · Mahesh J Salgaonkar
Quoted 65 lines · Avinash Roy
From: Shivaprasad G Bhat <[email protected]>
The IBM Spyre accelerator (PCI ID 1014:06a7) requires a dedicated warm reset method for reliable EEH recovery on pSeries/ppc64 platforms.
FLR alone is insufficient because it does not re-initialize firmware state machine for the current version of the card. The existing API
pci_set_pcie_reset_state()is used to drive the platform-level PERST# using the EEH interfaces by adding a device specific quirk.The reset function is placed in
arch/powerpc/platforms/pseries/pci.cwhere the EEH infrastructure is already present, declared inpseries.h, and registered in the generic pci_dev_reset_methods[] table indrivers/pci/quirks.cunder aCONFIG_PPC_PSERIESguard.This reset method is only registered when
CONFIG_PPC_PSERIESis enabled, so it is not available on other platforms.Reset sequence:
- Assert PERST# viapcie_warm_reset, hold 250 ms
- Deassert PERST# viapcie_deassert_reset
- Wait 250 ms for PCIe link retrainSigned-off-by Avinash Roy <[email protected]>Signed-off-by Shivaprasad G Bhat <[email protected]>---
Changelog
v1: https://lore.kernel.org/linux-pci/[email protected]/Reworded the commit message to highlight the warm reset requirement being a need of the current card and not a platform deficiency.
arch/powerpc/platforms/pseries/pci.c +17
-0@@ -11,6 +11,7 @@#include <linux/kernel.h> #include <linux/pci.h> #include <linux/string.h> +#include <linux/delay.h> #include <asm/eeh.h> #include <asm/pci-bridge.h> @@ -236,6 +237,41 @@ static void fixup_winbond_82c105(struct pci_dev* dev)DECLARE_PCI_FIXUP_HEADER (PCI_VENDOR_ID_WINBOND ,PCI_DEVICE_ID_WINBOND_82C105 , fixup_winbond_82c105); +/* +* Reset Spyre adapter using pci_set_pcie_reset_state() +* This is specifically for PPC platforms where EEH (Enhanced Error Handling) +* requires this reset method for proper device recovery. +*/ +int reset_spyre (struct pci_dev *dev,bool probe) +{ +int ret; + +if (probe) +return 0 ; + +/* Assert warm reset */ + ret =pci_set_pcie_reset_state (dev, pcie_warm_reset); +if (ret) { +pci_err (dev,"Failed to assert reset: %d\n" , ret);
Quoted 8 lines
arch/powerpc/platforms/pseries/pci.c static void fixup_winbond_82c105(struct pci_dev* dev) · line 260
Quoted 5 lines
arch/powerpc/platforms/pseries/pci.c +5
-0+return ret; + } + +/* Wait for reset to take effect */ +msleep (250 );How did you arriver on this number ? Is this time defined by hardware/firmware specs for reset ? Also, Can you #define the value 250 and use macro instead of constant numbmer.
This was the delay shared by the hardware team derived from their calibrations.
Thanks,
Avinash
Quoted 20 lines
arch/powerpc/platforms/pseries/pci.c static void fixup_winbond_82c105(struct pci_dev* dev) · line 270
Quoted 11 lines
arch/powerpc/platforms/pseries/pci.c +10
-0+ +/* Deassert reset */ + ret =pci_set_pcie_reset_state (dev, pcie_deassert_reset); +if (ret) { +pci_err (dev,"Failed to deassert reset: %d\n" , ret); +return ret; + } + +/* Wait for device to recover */ +msleep (250 );same here.
arch/powerpc/platforms/pseries/pci.c static void fixup_winbond_82c105(struct pci_dev* dev) · line 258
Quoted 4 lines
arch/powerpc/platforms/pseries/pci.c +4
-0+ +return 0 ; +} +Thanks,
-Mahesh.
Quoted 4 lines · Avinash Roy
From: Shivaprasad G Bhat <[email protected]>
The IBM Spyre accelerator (PCI ID 1014:06a7) requires a dedicated warm reset method for reliable EEH recovery on pSeries/ppc64 platforms.
It sounds like if you plugged a Spyre into an x86/arm64/etc platform, it would still need a dedicated warm reset quirk because the issue is with Spyre, not with pSeries.
Quoted 41 lines
FLR alone is insufficient because it does not re-initialize firmware state machine for the current version of the card. The existing API
pci_set_pcie_reset_state()is used to drive the platform-level PERST# using the EEH interfaces by adding a device specific quirk.The reset function is placed in
arch/powerpc/platforms/pseries/pci.cwhere the EEH infrastructure is already present, declared inpseries.h, and registered in the generic pci_dev_reset_methods[] table indrivers/pci/quirks.cunder aCONFIG_PPC_PSERIESguard.This reset method is only registered when
CONFIG_PPC_PSERIESis enabled, so it is not available on other platforms.Reset sequence:
- Assert PERST# viapcie_warm_reset, hold 250 ms
- Deassert PERST# viapcie_deassert_reset
- Wait 250 ms for PCIe link retrainSigned-off-by Avinash Roy <[email protected]>Signed-off-by Shivaprasad G Bhat <[email protected]>---
Changelog
v1: https://lore.kernel.org/linux-pci/[email protected]/Reworded the commit message to highlight the warm reset requirement being a need of the current card and not a platform deficiency.
arch/powerpc/platforms/pseries/pci.c +1
-0@@ -11,6 +11,7 @@#include <linux/kernel.h> #include <linux/pci.h> #include <linux/string.h> +#include <linux/delay.h> #include <asm/eeh.h> #include <asm/pci-bridge.h>
arch/powerpc/platforms/pseries/pci.c static void fixup_winbond_82c105(struct pci_dev* dev) · line 243
Quoted 7 lines
arch/powerpc/platforms/pseries/pci.c +4
-0@@ -236,6 +237,41 @@ static void fixup_winbond_82c105(struct pci_dev* dev)DECLARE_PCI_FIXUP_HEADER (PCI_VENDOR_ID_WINBOND ,PCI_DEVICE_ID_WINBOND_82C105 , fixup_winbond_82c105); +/* +* Reset Spyre adapter using pci_set_pcie_reset_state() +* This is specifically for PPC platforms where EEH (Enhanced Error Handling) +* requires this reset method for proper device recovery.
This comment still suggests that the issue is related to the platform, not the adapter itself. I know reset_spyre() will only *work* on pSeries because pcibios_set_pcie_reset_state() is only implemented there.
Quoted 54 lines
arch/powerpc/platforms/pseries/pci.c +31
-0+*/ +int reset_spyre (struct pci_dev *dev,bool probe) +{ +int ret; + +if (probe) +return 0 ; + +/* Assert warm reset */ + ret =pci_set_pcie_reset_state (dev, pcie_warm_reset); +if (ret) { +pci_err (dev,"Failed to assert reset: %d\n" , ret); +return ret; + } + +/* Wait for reset to take effect */ +msleep (250 ); + +/* Deassert reset */ + ret =pci_set_pcie_reset_state (dev, pcie_deassert_reset); +if (ret) { +pci_err (dev,"Failed to deassert reset: %d\n" , ret); +return ret; + } + +/* Wait for device to recover */ +msleep (250 ); + +return 0 ; +} +static enum pci_bus_speed prop_to_pci_speed (u32 prop) {switch (prop) {arch/powerpc/platforms/pseries/pseries.h +4
-0@@ -92,6 +92,10 @@ extern struct pci_controller_ops pseries_pci_controller_ops;int pseries_msi_allocate_domains (struct pci_controller *phb);void pseries_msi_free_domains (struct pci_controller *phb); +/* PCI device reset method for Spyre adapter */ +struct pci_dev ; +int reset_spyre (struct pci_dev *dev,bool probe); +extern int CMO_PrPSP;extern int CMO_SecPSP;extern unsigned long CMO_PageSize;drivers/pci/quirks.c +0
-0
drivers/pci/quirks.c static int reset_hinic_vf_dev(struct pci_dev *pdev, bool probe) · line 4243
Quoted 7 lines
drivers/pci/quirks.c +4
-0@@ -4237,6 +4237,11 @@ static int reset_hinic_vf_dev(struct pci_dev *pdev, bool probe)return 0 ; } +#ifdef CONFIG_PPC_PSERIES +/* Defined in arch/powerpc/platforms/pseries/pci.c */ +int reset_spyre (struct pci_dev *dev,bool probe); +#endif
Given that we need these ifdefs anyway, I think it would be simpler to just include the reset_spyre() implementation here. It doesn't look like it needs anything pSeries-specific, since pci_set_pcie_reset_state() is already available.
Quoted 13 lines
drivers/pci/quirks.c +4
-0static const struct pci_dev_reset_methods pci_dev_reset_methods[] = { {PCI_VENDOR_ID_INTEL ,PCI_DEVICE_ID_INTEL_82599_SFP_VF , reset_intel_82599_sfp_virtfn },@@ -4252,6 +4257,10 @@ static const struct pci_dev_reset_methods pci_dev_reset_methods[] = { reset_chelsio_generic_dev }, { PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HINIC_VF, reset_hinic_vf_dev }, +#ifdef CONFIG_PPC_PSERIES + { PCI_VENDOR_ID_IBM, 0x06a7, + reset_spyre}, +#endif { 0 } };