Skip to contentSkip to conversation

PCI: Work around CXL Port PM Init failure when ACS SV enabled

RFC PATCH Superseded Fabio M. De Francesco 2 participants 3 messages Mar 6 – Mar 7

Subsystems (2)

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 PCI: Work around CXL Port PM Init failure when ACS SV enabled

Compute Express Link (CXL) Specification Revision 4.0, Version 1.0, Section 8.1.5.1 - CXL Port Extension Status, Implementation Note, describes a scenario where a CXL downstream port may fail PM initialization:

"Certain conditions such as Link Down, Secondary Bus Reset, or Downstream Port Containment reset the Downstream Component’s bus number. If the Component generates the CREDIT_RTN IP2PM message with Requester Bus=0, the Downstream Port may reject the IP2PM message if software has enabled ACS Source Validation. In this scenario, Power Management initialization may fail to complete and another Secondary Bus Reset alone will not facilitate recovery".

Implement the recommended workaround for this scenario, which involves saving and disabling the ACS Source Validation and Bus Master Enable bits, issuing a secondary bus reset, waiting for PM init to complete, and then restoring those bits.

This is an RFC because it checks for PM init failures in ports enumeration, where an SBR might not be very welcome. Maybe other call sites are more appropriate for this workaround?

---
 drivers/cxl/core/pci.c        | 170 ++++++++++++++++++++++++++++++++++
 drivers/cxl/core/port.c       |  19 ++++
 drivers/cxl/cxlpci.h          |   3 +
 drivers/pci/pci.c             |   3 +-
 include/uapi/linux/pci_regs.h |   4 +
 5 files changed, 198 insertions(+), 1 deletion(-)
drivers/cxl/core/pci.c +170 -0
index f96ce884a213..76b69ff17b74 100644
@@ -869,3 +869,173 @@ int cxl_port_get_possible_dports(struct cxl_port *port)
 
 	return ctx.count;
 }
+
+struct pci_saved_flags {
+	u16 pci_command;
+	u16 acs_ctrl;
+};
+
+/*
+ * pci_disable_acs_bme - disable ACS Source Validation and BME
+ * @pdev: downstream port
+ * @saved: pointer to pci_saved_acs_bme struct, for saving bit values
+ *
+ * Save the current ACS Source Validation and Bus Master Enable (BME)
+ * bits before disabling them.
+ */
+static void pci_disable_acs_bme(struct pci_dev *pdev,
+				struct pci_saved_flags *saved)
+{
+	int pos;
+	u16 field;
+
+	pos = pdev->acs_cap;
+	if (!pos)
+		return;
+
+	pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &field);
+	saved->acs_ctrl = field;
+
+	if (field & PCI_ACS_SV)
+		pci_write_config_word(pdev, pos + PCI_ACS_CTRL,
+				      field & ~PCI_ACS_SV);
+
+	pci_read_config_word(pdev, PCI_COMMAND, &field);
+	saved->pci_command = field;
+
+	if (field & PCI_COMMAND_MASTER)
+		pci_clear_master(pdev);
+}
+
+/*
+ * pci_enable_acs_bme - enable ACS Source Validation and BME
+ * @pdev: downstream port
+ * @saved: pointer to pci_saved_acs_bme struct, for restoring bit values
+ *
+ * Restore the previously saved ACS Source Validation and Bus Master
+ * Enable (BME) bits.
+ */
+static void pci_enable_acs_bme(struct pci_dev *pdev,
+			       struct pci_saved_flags *saved)
+{
+	int pos;
+	u16 field;
+
+	pos = pdev->acs_cap;
+	if (!pos)
+		return;
+
+	pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &field);
+	if (saved->acs_ctrl & PCI_ACS_SV)
+		pci_write_config_word(pdev, pos + PCI_ACS_CTRL,
+				      field | PCI_ACS_SV);
+
+	pci_read_config_word(pdev, PCI_COMMAND, &field);
+	if (saved->pci_command & PCI_COMMAND_MASTER)
+		pci_set_master(pdev);
+}
+
+/**
+ * cxl_port_pm_init_is_complete - check if the downstream port has completed PM
+ * init
+ *
+ * @pdev: downstream port
+ *
+ * Check if the Port Power Management Initialization Complete bit is set in the
+ * Downstream Port's CXL DVSEC Port Extended Status register.
+ *
+ * Returns true if PM init is complete, false otherwise.
+ */
+bool cxl_port_pm_init_is_complete(struct pci_dev *pdev)
+{
+	int pm_init_complete;
+	u16 status;
+	u16 dvsec;
+
+	dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
+					  PCI_DVSEC_CXL_PORT);
+	if (!dvsec)
+		return false;
+
+	pci_read_config_word(pdev, dvsec + CXL_DVSEC_PORT_EXT_STATUS, &status);
+	pm_init_complete = FIELD_GET(CXL_DVSEC_PORT_EXT_STATUS_PM_INIT_COMP_MASK,
+				     status);
+
+	return !!pm_init_complete;
+}
+
+static void mask_sbr(struct pci_dev *pdev)
+{
+	int dvsec;
+	u16 reg;
+
+	dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
+					  PCI_DVSEC_CXL_PORT);
+	if (!dvsec)
+		return;
+
+	pci_read_config_word(pdev, dvsec + PCI_DVSEC_CXL_PORT_CTL, &reg);
+
+	pci_write_config_word(pdev, dvsec + PCI_DVSEC_CXL_PORT_CTL,
+			      reg & ~PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR);
+}
+
+static void unmask_sbr(struct pci_dev *pdev)
+{
+	int dvsec;
+	u16 reg;
+
+	dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
+					  PCI_DVSEC_CXL_PORT);
+	if (!dvsec)
+		return;
+
+	pci_read_config_word(pdev, dvsec + PCI_DVSEC_CXL_PORT_CTL, &reg);
+
+	pci_write_config_word(pdev, dvsec + PCI_DVSEC_CXL_PORT_CTL,
+			      reg | PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR);
+}
+
+/**
+ * cxl_port_retry_failed_pm_init - retry a downstream port's failed PM init
+ *
+ * @pdev: downstream port
+ *
+ * Retry a downstream CXL port's failed PM init according to CXL Erratum for
+ * Section 8.1.5.1 - Port Power Management Initialization Complete.
+ *
+ * This entails saving and disabling the ACS Source Validation and Bus Master
+ * enable bits, and unmasking and masking the SBR, before doing a secondary bus
+ * reset and then restoring and re-enabling those bits.
+ *
+ * return: 0 on success, errors otherwise.
+ */
+int cxl_port_retry_failed_pm_init(struct pci_dev *pdev)
+{
+	struct pci_saved_flags saved;
+	int ret;
+
+	if (!is_cxl_port(pdev->dev.parent))
+		return -EINVAL;
+
+	pci_disable_acs_bme(pdev, &saved);
+	unmask_sbr(pdev);
+
+	/* Generate Secondary Bus Reset */
+	ret = pci_reset_bus(pdev);
+	if (ret)
+		dev_dbg(&pdev->dev, "Bus reset failed: %d\n", ret);
+
+	/*
+	 * Wait until the Port Power Management Initialization
+	 * Complete bit is set in the Downstream Port. CXL Spec 4.0
+	 * Section 8.1.5.1 of the CXL spec specifies 100 ms as the
+	 * max time needed for the PM Init Complete bit to be set.
+	 */
+	msleep(100);
+
+	mask_sbr(pdev);
+	pci_enable_acs_bme(pdev, &saved);
+
+	return ret;
+}
drivers/cxl/core/port.c +19 -0
index b69c2529744c..62fabcdf0faf 100644
@@ -1829,12 +1829,31 @@ int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
 retry:
 	for (iter = dev; iter; iter = grandparent(iter)) {
 		struct device *dport_dev = grandparent(iter);
+		struct pci_dev *dport_pdev = to_pci_dev(dport_dev);
 		struct device *uport_dev;
 		struct cxl_dport *dport;
 
 		if (is_cxl_host_bridge(dport_dev))
 			return 0;
 
+		/*
+		 * Check the downstream port's PM init status, and if it has
+		 * failed retry PM init according to CXL Spec. 4.0 Sect. 8.1.5.1
+		 * - Implementation Note
+		 */
+		if (!cxl_port_pm_init_is_complete(dport_pdev)) {
+			dev_dbg(&cxlmd->dev,
+				"PM init failed for %s, retrying PM init\n",
+				dev_name(dport_dev));
+
+			cxl_port_retry_failed_pm_init(dport_pdev);
+
+			if (!cxl_port_pm_init_is_complete(dport_pdev))
+				dev_dbg(&cxlmd->dev,
+					"PM init failed retry for %s\n",
+					dev_name(dport_dev));
+		}
+
 		uport_dev = dport_dev->parent;
 		if (!uport_dev) {
 			dev_warn(dev, "at %s no parent for dport: %s\n",
drivers/cxl/cxlpci.h +3 -0
index 0cf64218aa16..0458096be2c4 100644
@@ -101,4 +101,7 @@ static inline void devm_cxl_port_ras_setup(struct cxl_port *port)
 }
 #endif
 
+bool cxl_port_pm_init_is_complete(struct pci_dev *pdev);
+int cxl_port_retry_failed_pm_init(struct pci_dev *pdev);
+
 #endif /* __CXL_PCI_H__ */
drivers/pci/pci.c +2 -1
index 8479c2e1f74f..511fc9e70e2a 100644
@@ -4881,7 +4881,7 @@ static u16 cxl_port_dvsec(struct pci_dev *dev)
 					 PCI_DVSEC_CXL_PORT);
 }
 
-static bool cxl_sbr_masked(struct pci_dev *dev)
+bool cxl_sbr_masked(struct pci_dev *dev)
 {
 	u16 dvsec, reg;
 	int rc;
@@ -4904,6 +4904,7 @@ static bool cxl_sbr_masked(struct pci_dev *dev)
 
 	return true;
 }
+EXPORT_SYMBOL_NS_GPL(cxl_sbr_masked, "CXL");
 
 static int pci_reset_bus_function(struct pci_dev *dev, bool probe)
 {
include/uapi/linux/pci_regs.h +4 -0
index ec1c54b5a310..1a12552fa89a 100644
@@ -1397,4 +1397,8 @@
 #define   PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_ID		__GENMASK(15, 8)
 #define   PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_OFF_LOW	__GENMASK(31, 16)
 
+/* CXL 4.0 8.1.5.1: CXL Port Extension Status */
+#define CXL_DVSEC_PORT_EXT_STATUS 0xA
+#define CXL_DVSEC_PORT_EXT_STATUS_PM_INIT_COMP_MASK GENMASK(1, 0)
+
 #endif /* LINUX_PCI_REGS_H */
Signature · 1 line
2.53.0
#1 lore raw
kernel test robot noticed the following build warnings:

Hi Fabio,

[This is a private test report for your RFC patch.] kernel test robot noticed the following build warnings:

[auto build test WARNING on pci/next]
[also build test WARNING on pci/for-linus cxl/next linus/master v7.0-rc2 next-20260306] [cannot apply to cxl/pending]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Fabio-M-De-Francesco/PCI-Work-around-CXL-Port-PM-Init-failure-when-ACS-SV-enabled/20260307-035826
base:   https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git next
patch link:    https://lore.kernel.org/r/20260306195232.388814-1-fabio.m.de.francesco%40linux.intel.com
patch subject: [RFC PATCH] PCI: Work around CXL Port PM Init failure when ACS SV enabled
config: alpha-allnoconfig (https://download.01.org/0day-ci/archive/20260307/[email protected]/config)
compiler: alpha-linux-gcc (GCC) 15.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260307/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All warnings (new ones prefixed by >>):

>> drivers/pci/pci.c:4883:6: warning: no previous prototype for 'cxl_sbr_masked' [-Wmissing-prototypes]

4883 | bool cxl_sbr_masked(struct pci_dev *dev)
| ^~~~~~~~~~~~~~

vim +/cxl_sbr_masked +4883 drivers/pci/pci.c

4882

> 4883	bool cxl_sbr_masked(struct pci_dev *dev)

4884 {
4885 u16 dvsec, reg;
4886 int rc;
4887
4888 dvsec = cxl_port_dvsec(dev);
4889 if (!dvsec)
4890 return false;
4891
4892 rc = pci_read_config_word(dev, dvsec + PCI_DVSEC_CXL_PORT_CTL, &reg);
4893 if (rc || PCI_POSSIBLE_ERROR(reg))
4894 return false;
4895
4896 /*
4897 * Per CXL spec r3.1, sec 8.1.5.2, when "Unmask SBR" is 0, the SBR
4898 * bit in Bridge Control has no effect. When 1, the Port generates
4899 * hot reset when the SBR bit is set to 1.
4900 */
4901 if (reg & PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR)
4902 return false;
4903
4904 return true;
4905 }
4906 EXPORT_SYMBOL_NS_GPL(cxl_sbr_masked, "CXL");
4907

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
#2 lore raw
In reply to Fabio M. De Francesco
kernel test robot noticed the following build warnings:

Hi Fabio,

[This is a private test report for your RFC patch.] kernel test robot noticed the following build warnings:

[auto build test WARNING on pci/next]
[also build test WARNING on pci/for-linus cxl/next linus/master v7.0-rc2 next-20260305] [cannot apply to cxl/pending]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Fabio-M-De-Francesco/PCI-Work-around-CXL-Port-PM-Init-failure-when-ACS-SV-enabled/20260307-035826
base:   https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git next
patch link:    https://lore.kernel.org/r/20260306195232.388814-1-fabio.m.de.francesco%40linux.intel.com
patch subject: [RFC PATCH] PCI: Work around CXL Port PM Init failure when ACS SV enabled
config: loongarch-allnoconfig (https://download.01.org/0day-ci/archive/20260307/[email protected]/config)
compiler: clang version 23.0.0git (https://github.com/llvm/llvm-project c32caeec8158d634bb71ab8911a6031248b9fc47)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260307/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All warnings (new ones prefixed by >>):

>> drivers/pci/pci.c:4883:6: warning: no previous prototype for function 'cxl_sbr_masked' [-Wmissing-prototypes]
    4883 | bool cxl_sbr_masked(struct pci_dev *dev)
         |      ^
   drivers/pci/pci.c:4883:1: note: declare 'static' if the function is not intended to be used outside of this translation unit
    4883 | bool cxl_sbr_masked(struct pci_dev *dev)
         | ^
         | static 

1 warning generated.

vim +/cxl_sbr_masked +4883 drivers/pci/pci.c

4882

> 4883	bool cxl_sbr_masked(struct pci_dev *dev)

4884 {
4885 u16 dvsec, reg;
4886 int rc;
4887
4888 dvsec = cxl_port_dvsec(dev);
4889 if (!dvsec)
4890 return false;
4891
4892 rc = pci_read_config_word(dev, dvsec + PCI_DVSEC_CXL_PORT_CTL, &reg);
4893 if (rc || PCI_POSSIBLE_ERROR(reg))
4894 return false;
4895
4896 /*
4897 * Per CXL spec r3.1, sec 8.1.5.2, when "Unmask SBR" is 0, the SBR
4898 * bit in Bridge Control has no effect. When 1, the Port generates
4899 * hot reset when the SBR bit is set to 1.
4900 */
4901 if (reg & PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR)
4902 return false;
4903
4904 return true;
4905 }
4906 EXPORT_SYMBOL_NS_GPL(cxl_sbr_masked, "CXL");
4907

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki