Skip to contentSkip to conversation

PCI: quirks: Add warm reset quirk for IBM Spyre accelerator

PATCH In review Avinash Roy 3 participants 4 messages Aug 21 – Oct 9

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 PCI: quirks: Add warm reset quirk for IBM Spyre accelerator

From: Shivaprasad G Bhat <[email protected]>

The IBM Spyre accelerator (PCI ID 1014:06a7) requires a dedicated warm reset method for reliable EEH recovery on pSeries/ppc64 platforms.

FLR alone is insufficient because it does not re-initialize firmware state machine for the current version of the card. The existing API pci_set_pcie_reset_state() is used to drive the platform-level PERST# using the EEH interfaces by adding a device specific quirk.

The reset function is placed in arch/powerpc/platforms/pseries/pci.c where the EEH infrastructure is already present, declared in pseries.h, and registered in the generic pci_dev_reset_methods[] table in drivers/pci/quirks.c under a CONFIG_PPC_PSERIES guard.

This reset method is only registered when CONFIG_PPC_PSERIES is enabled, so it is not available on other platforms.

Reset sequence:
- Assert PERST# via pcie_warm_reset, hold 250 ms
- Deassert PERST# via pcie_deassert_reset
- Wait 250 ms for PCIe link retrain

---
Changelog
v1: https://lore.kernel.org/linux-pci/[email protected]/

Reworded the commit message to highlight the warm reset requirement being a need of the current card and not a platform deficiency.

arch/powerpc/platforms/pseries/pci.c +36 -0
index 84e4ffe957a8..7aa58d57208f 100644
@@ -11,6 +11,7 @@
 #include <linux/kernel.h>
 #include <linux/pci.h>
 #include <linux/string.h>
+#include <linux/delay.h>
 
 #include <asm/eeh.h>
 #include <asm/pci-bridge.h>
@@ -236,6 +237,41 @@ static void fixup_winbond_82c105(struct pci_dev* dev)
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_WINBOND, PCI_DEVICE_ID_WINBOND_82C105,
 			 fixup_winbond_82c105);
 
+/*
+ * Reset Spyre adapter using pci_set_pcie_reset_state()
+ * This is specifically for PPC platforms where EEH (Enhanced Error Handling)
+ * requires this reset method for proper device recovery.
+ */
+int reset_spyre(struct pci_dev *dev, bool probe)
+{
+	int ret;
+
+	if (probe)
+		return 0;
+
+	/* Assert warm reset */
+	ret = pci_set_pcie_reset_state(dev, pcie_warm_reset);
+	if (ret) {
+		pci_err(dev, "Failed to assert reset: %d\n", ret);
+		return ret;
+	}
+
+	/* Wait for reset to take effect */
+	msleep(250);
+
+	/* Deassert reset */
+	ret = pci_set_pcie_reset_state(dev, pcie_deassert_reset);
+	if (ret) {
+		pci_err(dev, "Failed to deassert reset: %d\n", ret);
+		return ret;
+	}
+
+	/* Wait for device to recover */
+	msleep(250);
+
+	return 0;
+}
+
 static enum pci_bus_speed prop_to_pci_speed(u32 prop)
 {
 	switch (prop) {
arch/powerpc/platforms/pseries/pseries.h +4 -0
index 3968a6970fa8..79c194aeb3c6 100644
@@ -92,6 +92,10 @@ extern struct pci_controller_ops pseries_pci_controller_ops;
 int pseries_msi_allocate_domains(struct pci_controller *phb);
 void pseries_msi_free_domains(struct pci_controller *phb);
 
+/* PCI device reset method for Spyre adapter */
+struct pci_dev;
+int reset_spyre(struct pci_dev *dev, bool probe);
+
 extern int CMO_PrPSP;
 extern int CMO_SecPSP;
 extern unsigned long CMO_PageSize;
drivers/pci/quirks.c +9 -0
index b09f27f7846f..1b8ab5d9764b 100644
@@ -4237,6 +4237,11 @@ static int reset_hinic_vf_dev(struct pci_dev *pdev, bool probe)
 	return 0;
 }
 
+#ifdef CONFIG_PPC_PSERIES
+/* Defined in arch/powerpc/platforms/pseries/pci.c */
+int reset_spyre(struct pci_dev *dev, bool probe);
+#endif
+
 static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
 	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82599_SFP_VF,
 		 reset_intel_82599_sfp_virtfn },
@@ -4252,6 +4257,10 @@ static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
 		reset_chelsio_generic_dev },
 	{ PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HINIC_VF,
 		reset_hinic_vf_dev },
+#ifdef CONFIG_PPC_PSERIES
+	{ PCI_VENDOR_ID_IBM, 0x06a7,
+		reset_spyre},
+#endif
 	{ 0 }
 };
#1 lore raw
Quoted 60 lines · Avinash Roy

From: Shivaprasad G Bhat <[email protected]>

The IBM Spyre accelerator (PCI ID 1014:06a7) requires a dedicated warm reset method for reliable EEH recovery on pSeries/ppc64 platforms.

FLR alone is insufficient because it does not re-initialize firmware state machine for the current version of the card. The existing API pci_set_pcie_reset_state() is used to drive the platform-level PERST# using the EEH interfaces by adding a device specific quirk.

The reset function is placed in arch/powerpc/platforms/pseries/pci.c where the EEH infrastructure is already present, declared in pseries.h, and registered in the generic pci_dev_reset_methods[] table in drivers/pci/quirks.c under a CONFIG_PPC_PSERIES guard.

This reset method is only registered when CONFIG_PPC_PSERIES is enabled, so it is not available on other platforms.

Reset sequence:
- Assert PERST# via pcie_warm_reset, hold 250 ms
- Deassert PERST# via pcie_deassert_reset
- Wait 250 ms for PCIe link retrain

---
Changelog
v1: https://lore.kernel.org/linux-pci/[email protected]/

Reworded the commit message to highlight the warm reset requirement being a need of the current card and not a platform deficiency.

arch/powerpc/platforms/pseries/pci.c +12 -0
index 84e4ffe957a8..7aa58d57208f 100644
@@ -11,6 +11,7 @@
 #include <linux/kernel.h>
 #include <linux/pci.h>
 #include <linux/string.h>
+#include <linux/delay.h>
 
 #include <asm/eeh.h>
 #include <asm/pci-bridge.h>
@@ -236,6 +237,41 @@ static void fixup_winbond_82c105(struct pci_dev* dev)
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_WINBOND, PCI_DEVICE_ID_WINBOND_82C105,
 			 fixup_winbond_82c105);
 
+/*
+ * Reset Spyre adapter using pci_set_pcie_reset_state()
+ * This is specifically for PPC platforms where EEH (Enhanced Error Handling)
+ * requires this reset method for proper device recovery.
+ */
+int reset_spyre(struct pci_dev *dev, bool probe)
+{
+	int ret;
+
+	if (probe)
+		return 0;

arch/powerpc/platforms/pseries/pci.c static void fixup_winbond_82c105(struct pci_dev* dev) · line 260

Quoted 10 lines
arch/powerpc/platforms/pseries/pci.c +10 -0
+
+	/* Assert warm reset */
+	ret = pci_set_pcie_reset_state(dev, pcie_warm_reset);
+	if (ret) {
+		pci_err(dev, "Failed to assert reset: %d\n", ret);
+		return ret;
+	}
+
+	/* Wait for reset to take effect */
+	msleep(250);

How did you arriver on this number ? Is this time defined by hardware/firmware specs for reset ? Also, Can you #define the value 250 and use macro instead of constant numbmer.

arch/powerpc/platforms/pseries/pci.c static void fixup_winbond_82c105(struct pci_dev* dev) · line 270

Quoted 10 lines
arch/powerpc/platforms/pseries/pci.c +10 -0
+
+	/* Deassert reset */
+	ret = pci_set_pcie_reset_state(dev, pcie_deassert_reset);
+	if (ret) {
+		pci_err(dev, "Failed to deassert reset: %d\n", ret);
+		return ret;
+	}
+
+	/* Wait for device to recover */
+	msleep(250);

same here.

arch/powerpc/platforms/pseries/pci.c static void fixup_winbond_82c105(struct pci_dev* dev) · line 258

Quoted 4 lines
arch/powerpc/platforms/pseries/pci.c +4 -0
+
+	return 0;
+}
+

Thanks,
-Mahesh.

#2 lore raw
Quoted 66 lines · Mahesh J Salgaonkar
Quoted 65 lines · Avinash Roy

From: Shivaprasad G Bhat <[email protected]>

The IBM Spyre accelerator (PCI ID 1014:06a7) requires a dedicated warm reset method for reliable EEH recovery on pSeries/ppc64 platforms.

FLR alone is insufficient because it does not re-initialize firmware state machine for the current version of the card. The existing API pci_set_pcie_reset_state() is used to drive the platform-level PERST# using the EEH interfaces by adding a device specific quirk.

The reset function is placed in arch/powerpc/platforms/pseries/pci.c where the EEH infrastructure is already present, declared in pseries.h, and registered in the generic pci_dev_reset_methods[] table in drivers/pci/quirks.c under a CONFIG_PPC_PSERIES guard.

This reset method is only registered when CONFIG_PPC_PSERIES is enabled, so it is not available on other platforms.

Reset sequence:
- Assert PERST# via pcie_warm_reset, hold 250 ms
- Deassert PERST# via pcie_deassert_reset
- Wait 250 ms for PCIe link retrain

---
Changelog
v1: https://lore.kernel.org/linux-pci/[email protected]/

Reworded the commit message to highlight the warm reset requirement being a need of the current card and not a platform deficiency.

arch/powerpc/platforms/pseries/pci.c +17 -0
index 84e4ffe957a8..7aa58d57208f 100644
@@ -11,6 +11,7 @@
  #include <linux/kernel.h>
  #include <linux/pci.h>
  #include <linux/string.h>
+#include <linux/delay.h>
 
  #include <asm/eeh.h>
  #include <asm/pci-bridge.h>
@@ -236,6 +237,41 @@ static void fixup_winbond_82c105(struct pci_dev* dev)
  DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_WINBOND, PCI_DEVICE_ID_WINBOND_82C105,
  			 fixup_winbond_82c105);
 
+/*
+ * Reset Spyre adapter using pci_set_pcie_reset_state()
+ * This is specifically for PPC platforms where EEH (Enhanced Error Handling)
+ * requires this reset method for proper device recovery.
+ */
+int reset_spyre(struct pci_dev *dev, bool probe)
+{
+	int ret;
+
+	if (probe)
+		return 0;
+
+	/* Assert warm reset */
+	ret = pci_set_pcie_reset_state(dev, pcie_warm_reset);
+	if (ret) {
+		pci_err(dev, "Failed to assert reset: %d\n", ret);
Quoted 8 lines

arch/powerpc/platforms/pseries/pci.c static void fixup_winbond_82c105(struct pci_dev* dev) · line 260

Quoted 5 lines
arch/powerpc/platforms/pseries/pci.c +5 -0
+		return ret;
+	}
+
+	/* Wait for reset to take effect */
+	msleep(250);

How did you arriver on this number ? Is this time defined by hardware/firmware specs for reset ? Also, Can you #define the value 250 and use macro instead of constant numbmer.

This was the delay shared by the hardware team derived from their calibrations.

Thanks,
Avinash

Quoted 20 lines

arch/powerpc/platforms/pseries/pci.c static void fixup_winbond_82c105(struct pci_dev* dev) · line 270

Quoted 11 lines
arch/powerpc/platforms/pseries/pci.c +10 -0
+
+	/* Deassert reset */
+	ret = pci_set_pcie_reset_state(dev, pcie_deassert_reset);
+	if (ret) {
+		pci_err(dev, "Failed to deassert reset: %d\n", ret);
+		return ret;
+	}
+
+	/* Wait for device to recover */
+	msleep(250);

same here.

arch/powerpc/platforms/pseries/pci.c static void fixup_winbond_82c105(struct pci_dev* dev) · line 258

Quoted 4 lines
arch/powerpc/platforms/pseries/pci.c +4 -0
+
+	return 0;
+}
+

Thanks,
-Mahesh.

#3 lore raw
In reply to Avinash Roy
Quoted 4 lines · Avinash Roy

From: Shivaprasad G Bhat <[email protected]>

The IBM Spyre accelerator (PCI ID 1014:06a7) requires a dedicated warm reset method for reliable EEH recovery on pSeries/ppc64 platforms.

It sounds like if you plugged a Spyre into an x86/arm64/etc platform, it would still need a dedicated warm reset quirk because the issue is with Spyre, not with pSeries.

Quoted 41 lines

FLR alone is insufficient because it does not re-initialize firmware state machine for the current version of the card. The existing API pci_set_pcie_reset_state() is used to drive the platform-level PERST# using the EEH interfaces by adding a device specific quirk.

The reset function is placed in arch/powerpc/platforms/pseries/pci.c where the EEH infrastructure is already present, declared in pseries.h, and registered in the generic pci_dev_reset_methods[] table in drivers/pci/quirks.c under a CONFIG_PPC_PSERIES guard.

This reset method is only registered when CONFIG_PPC_PSERIES is enabled, so it is not available on other platforms.

Reset sequence:
- Assert PERST# via pcie_warm_reset, hold 250 ms
- Deassert PERST# via pcie_deassert_reset
- Wait 250 ms for PCIe link retrain

---
Changelog
v1: https://lore.kernel.org/linux-pci/[email protected]/

Reworded the commit message to highlight the warm reset requirement being a need of the current card and not a platform deficiency.

arch/powerpc/platforms/pseries/pci.c +1 -0
index 84e4ffe957a8..7aa58d57208f 100644
@@ -11,6 +11,7 @@
 #include <linux/kernel.h>
 #include <linux/pci.h>
 #include <linux/string.h>
+#include <linux/delay.h>
 
 #include <asm/eeh.h>
 #include <asm/pci-bridge.h>

arch/powerpc/platforms/pseries/pci.c static void fixup_winbond_82c105(struct pci_dev* dev) · line 243

Quoted 7 lines
arch/powerpc/platforms/pseries/pci.c +4 -0
@@ -236,6 +237,41 @@ static void fixup_winbond_82c105(struct pci_dev* dev)
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_WINBOND, PCI_DEVICE_ID_WINBOND_82C105,
 			 fixup_winbond_82c105);
 
+/*
+ * Reset Spyre adapter using pci_set_pcie_reset_state()
+ * This is specifically for PPC platforms where EEH (Enhanced Error Handling)
+ * requires this reset method for proper device recovery.

This comment still suggests that the issue is related to the platform, not the adapter itself. I know reset_spyre() will only *work* on pSeries because pcibios_set_pcie_reset_state() is only implemented there.

Quoted 54 lines
arch/powerpc/platforms/pseries/pci.c +31 -0
+ */
+int reset_spyre(struct pci_dev *dev, bool probe)
+{
+	int ret;
+
+	if (probe)
+		return 0;
+
+	/* Assert warm reset */
+	ret = pci_set_pcie_reset_state(dev, pcie_warm_reset);
+	if (ret) {
+		pci_err(dev, "Failed to assert reset: %d\n", ret);
+		return ret;
+	}
+
+	/* Wait for reset to take effect */
+	msleep(250);
+
+	/* Deassert reset */
+	ret = pci_set_pcie_reset_state(dev, pcie_deassert_reset);
+	if (ret) {
+		pci_err(dev, "Failed to deassert reset: %d\n", ret);
+		return ret;
+	}
+
+	/* Wait for device to recover */
+	msleep(250);
+
+	return 0;
+}
+
 static enum pci_bus_speed prop_to_pci_speed(u32 prop)
 {
 	switch (prop) {
arch/powerpc/platforms/pseries/pseries.h +4 -0
index 3968a6970fa8..79c194aeb3c6 100644
@@ -92,6 +92,10 @@ extern struct pci_controller_ops pseries_pci_controller_ops;
 int pseries_msi_allocate_domains(struct pci_controller *phb);
 void pseries_msi_free_domains(struct pci_controller *phb);
 
+/* PCI device reset method for Spyre adapter */
+struct pci_dev;
+int reset_spyre(struct pci_dev *dev, bool probe);
+
 extern int CMO_PrPSP;
 extern int CMO_SecPSP;
 extern unsigned long CMO_PageSize;
drivers/pci/quirks.c +0 -0
index b09f27f7846f..1b8ab5d9764b 100644

drivers/pci/quirks.c static int reset_hinic_vf_dev(struct pci_dev *pdev, bool probe) · line 4243

Quoted 7 lines
drivers/pci/quirks.c +4 -0
@@ -4237,6 +4237,11 @@ static int reset_hinic_vf_dev(struct pci_dev *pdev, bool probe)
 	return 0;
 }
 
+#ifdef CONFIG_PPC_PSERIES
+/* Defined in arch/powerpc/platforms/pseries/pci.c */
+int reset_spyre(struct pci_dev *dev, bool probe);
+#endif

Given that we need these ifdefs anyway, I think it would be simpler to just include the reset_spyre() implementation here. It doesn't look like it needs anything pSeries-specific, since pci_set_pcie_reset_state() is already available.

Quoted 13 lines
drivers/pci/quirks.c +4 -0
 static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
 	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82599_SFP_VF,
 		 reset_intel_82599_sfp_virtfn },
@@ -4252,6 +4257,10 @@ static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
 		reset_chelsio_generic_dev },
 	{ PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HINIC_VF,
 		reset_hinic_vf_dev },
+#ifdef CONFIG_PPC_PSERIES
+	{ PCI_VENDOR_ID_IBM, 0x06a7,
+		reset_spyre},
+#endif
 	{ 0 }
 };