Skip to contentSkip to conversation

PCI: dwc: Implement general suspend/resume functionality for L2/L3 transitions

PATCH Superseded Frank Li 4 participants 9 messages Jul 24, 2023 – Jul 31, 2023

Subsystems (5)

v5 of 7 Superseded Latest is v7 →

lore
Share & tools
lore

Frank Li

author maintainer
#0 lore raw

PATCH v5 1/2 PCI: dwc: Implement general suspend/resume functionality for L2/L3 transitions

Introduced helper function dw_pcie_get_ltssm to retrieve SMLH_LTSS_STATE. Added API pme_turn_off and exit_from_l2 for managing L2/L3 state transitions.

Typical L2 entry workflow:

1. Transmit PME turn off signal to PCI devices and wait for PME_To_Ack.
2. Await link entering L2_IDLE state.
3. Transition Root complex to D3 state.

Typical L2 exit workflow:

1. Transition Root complex to D0 state.
2. Issue exit from L2 command.
3. Reinitialize PCI host.
4. Wait for link to become active.

---
Change from v4 to v5:
- Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
Change from v3 to v4:
- change according to Manivannan's comments.
I hope I have not missed anything. quite long discuss thread
Change from v2 to v3:
- Basic rewrite whole patch according rob herry suggestion.
put common function into dwc, so more soc can share the same logic.

 .../pci/controller/dwc/pcie-designware-host.c | 95 +++++++++++++++++++
 drivers/pci/controller/dwc/pcie-designware.h  | 28 ++++++
 2 files changed, 123 insertions(+)
drivers/pci/controller/dwc/pcie-designware-host.c +95 -0
index 9952057c8819..031e1f9c0d0c 100644
@@ -8,6 +8,7 @@
  * Author: Jingoo Han <[email protected]>
  */
 
+#include <linux/iopoll.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/irqdomain.h>
 #include <linux/msi.h>
@@ -807,3 +808,97 @@ int dw_pcie_setup_rc(struct dw_pcie_rp *pp)
 	return 0;
 }
 EXPORT_SYMBOL_GPL(dw_pcie_setup_rc);
+
+/*
+ * This resemble the pci_set_power_state() interfaces, but these are for
+ * configuring host controllers, which are bridges *to* PCI devices but
+ * are not PCI devices themselves.
+ */
+static void dw_pcie_set_dstate(struct dw_pcie *pci, pci_power_t dstate)
+{
+	u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_PM);
+	u16 val;
+
+	val = dw_pcie_readw_dbi(pci, offset + PCI_PM_CTRL);
+	val &= ~PCI_PM_CTRL_STATE_MASK;
+	val |= ((u16 __force)dstate) & PCI_PM_CTRL_STATE_MASK;
+	dw_pcie_writew_dbi(pci, offset + PCI_PM_CTRL, val);
+}
+
+int dw_pcie_suspend_noirq(struct dw_pcie *pci)
+{
+	u8 offset;
+	u32 val;
+	int ret;
+
+	offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
+	/*
+	 * If L1.1\L1.2 enable, devices (such as NVME) want short
+	 * resume latency, controller will not enter L2
+	 */
+	if (dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKCTL) & PCI_EXP_LNKCTL_ASPM_L1)
+		return 0;
+
+	if (dw_pcie_get_ltssm(pci) <= DW_PCIE_LTSSM_DETECT_ACT)
+		return 0;
+
+	if (!pci->pp.ops->pme_turn_off)
+		return -EINVAL;
+
+	pci->pp.ops->pme_turn_off(&pci->pp);
+
+	/*
+	 * PCI Express Base Specification Rev 4.0
+	 * 5.3.3.2.1 PME Synchronization
+	 * Recommand 1ms to 10ms timeout to check L2 ready
+	 */
+	ret = read_poll_timeout(dw_pcie_get_ltssm, val, val == DW_PCIE_LTSSM_L2_IDLE,
+				100, 10000, false, pci);
+	if (ret) {
+		dev_err(pci->dev, "PCIe link enter L2 timeout! ltssm = 0x%x\n", val);
+		return ret;
+	}
+
+	dw_pcie_set_dstate(pci, PCI_D3hot);
+
+	pci->suspended = true;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(dw_pcie_suspend_noirq);
+
+int dw_pcie_resume_noirq(struct dw_pcie *pci)
+{
+	int ret;
+
+	if (!pci->suspended)
+		return 0;
+
+	pci->suspended = false;
+
+	dw_pcie_set_dstate(pci, PCI_D0);
+
+	if (!pci->pp.ops->exit_from_l2)
+		return -EINVAL;
+
+	pci->pp.ops->exit_from_l2(&pci->pp);
+
+	ret = pci->pp.ops->host_init(&pci->pp);
+	if (ret) {
+		dev_err(pci->dev, "Host init failed! ret = 0x%x\n", ret);
+		return ret;
+	}
+
+	dw_pcie_setup_rc(&pci->pp);
+
+	ret = dw_pcie_start_link(pci);
+	if (ret)
+		return ret;
+
+	ret = dw_pcie_wait_for_link(pci);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(dw_pcie_resume_noirq);
drivers/pci/controller/dwc/pcie-designware.h +28 -0
index 79713ce075cc..effb07a506e4 100644
@@ -288,10 +288,21 @@ enum dw_pcie_core_rst {
 	DW_PCIE_NUM_CORE_RSTS
 };
 
+enum dw_pcie_ltssm {
+	DW_PCIE_LTSSM_UNKNOWN = 0xFFFFFFFF,
+	/* Need align PCIE_PORT_DEBUG0 bit0:5 */
+	DW_PCIE_LTSSM_DETECT_QUIET = 0x0,
+	DW_PCIE_LTSSM_DETECT_ACT = 0x1,
+	DW_PCIE_LTSSM_L0 = 0x11,
+	DW_PCIE_LTSSM_L2_IDLE = 0x15,
+};
+
 struct dw_pcie_host_ops {
 	int (*host_init)(struct dw_pcie_rp *pp);
 	void (*host_deinit)(struct dw_pcie_rp *pp);
 	int (*msi_host_init)(struct dw_pcie_rp *pp);
+	void (*pme_turn_off)(struct dw_pcie_rp *pp);
+	void (*exit_from_l2)(struct dw_pcie_rp *pp);
 };
 
 struct dw_pcie_rp {
@@ -364,6 +375,7 @@ struct dw_pcie_ops {
 	void    (*write_dbi2)(struct dw_pcie *pcie, void __iomem *base, u32 reg,
 			      size_t size, u32 val);
 	int	(*link_up)(struct dw_pcie *pcie);
+	enum dw_pcie_ltssm (*get_ltssm)(struct dw_pcie *pcie);
 	int	(*start_link)(struct dw_pcie *pcie);
 	void	(*stop_link)(struct dw_pcie *pcie);
 };
@@ -393,6 +405,7 @@ struct dw_pcie {
 	struct reset_control_bulk_data	app_rsts[DW_PCIE_NUM_APP_RSTS];
 	struct reset_control_bulk_data	core_rsts[DW_PCIE_NUM_CORE_RSTS];
 	struct gpio_desc		*pe_rst;
+	bool			suspended;
 };
 
 #define to_dw_pcie_from_pp(port) container_of((port), struct dw_pcie, pp)
@@ -430,6 +443,9 @@ void dw_pcie_iatu_detect(struct dw_pcie *pci);
 int dw_pcie_edma_detect(struct dw_pcie *pci);
 void dw_pcie_edma_remove(struct dw_pcie *pci);
 
+int dw_pcie_suspend_noirq(struct dw_pcie *pci);
+int dw_pcie_resume_noirq(struct dw_pcie *pci);
+
 static inline void dw_pcie_writel_dbi(struct dw_pcie *pci, u32 reg, u32 val)
 {
 	dw_pcie_write_dbi(pci, reg, 0x4, val);
@@ -501,6 +517,18 @@ static inline void dw_pcie_stop_link(struct dw_pcie *pci)
 		pci->ops->stop_link(pci);
 }
 
+static inline enum dw_pcie_ltssm dw_pcie_get_ltssm(struct dw_pcie *pci)
+{
+	u32 val;
+
+	if (pci->ops && pci->ops->get_ltssm)
+		return pci->ops->get_ltssm(pci);
+
+	val = dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0);
+
+	return (enum dw_pcie_ltssm)FIELD_GET(PORT_LOGIC_LTSSM_STATE_MASK, val);
+}
+
 #ifdef CONFIG_PCIE_DW_HOST
 irqreturn_t dw_handle_msi_irq(struct dw_pcie_rp *pp);
 int dw_pcie_setup_rc(struct dw_pcie_rp *pp);
Signature · 1 line
2.34.1
author maintainer
#1 lore raw

PATCH v5 2/2 PCI: layerscape: Add power management support for ls1028a

From: Hou Zhiqiang <[email protected]>

Add PME_Turn_Off/PME_TO_Ack handshake sequence, and finally put the PCIe controller into D3 state after the L2/L3 ready state transition process completion.

---
- Change from v3 to v5
none

- change at v3
Basic rewrite whole patch according rob herry suggestion. put common function into dwc, so more soc can share the same logic.

 drivers/pci/controller/dwc/pci-layerscape.c | 148 ++++++++++++++++++--
 1 file changed, 139 insertions(+), 9 deletions(-)
drivers/pci/controller/dwc/pci-layerscape.c +139 -9
index ed5fb492fe08..32832d74a074 100644
@@ -8,9 +8,11 @@
  * Author: Minghuan Lian <[email protected]>
  */
 
+#include <linux/delay.h>
 #include <linux/kernel.h>
 #include <linux/interrupt.h>
 #include <linux/init.h>
+#include <linux/iopoll.h>
 #include <linux/of_pci.h>
 #include <linux/of_platform.h>
 #include <linux/of_address.h>
@@ -27,12 +29,33 @@
 #define PCIE_ABSERR		0x8d0 /* Bridge Slave Error Response Register */
 #define PCIE_ABSERR_SETTING	0x9401 /* Forward error of non-posted request */
 
+/* PF Message Command Register */
+#define LS_PCIE_PF_MCR		0x2c
+#define PF_MCR_PTOMR		BIT(0)
+#define PF_MCR_EXL2S		BIT(1)
+
 #define PCIE_IATU_NUM		6
 
+struct ls_pcie;
+
+struct ls_pcie_drvdata {
+	const u32 pf_off;
+	const u32 lut_off;
+	bool pm_support;
+};
+
 struct ls_pcie {
 	struct dw_pcie *pci;
+	const struct ls_pcie_drvdata *drvdata;
+	void __iomem *pf_base;
+	void __iomem *lut_base;
+	bool big_endian;
+	bool ep_presence;
+	struct regmap *scfg;
+	int index;
 };
 
+#define ls_pcie_pf_readl_addr(addr)	ls_pcie_pf_readl(pcie, addr)
 #define to_ls_pcie(x)	dev_get_drvdata((x)->dev)
 
 static bool ls_pcie_is_bridge(struct ls_pcie *pcie)
@@ -73,6 +96,57 @@ static void ls_pcie_fix_error_response(struct ls_pcie *pcie)
 	iowrite32(PCIE_ABSERR_SETTING, pci->dbi_base + PCIE_ABSERR);
 }
 
+static u32 ls_pcie_pf_readl(struct ls_pcie *pcie, u32 off)
+{
+	if (pcie->big_endian)
+		return ioread32be(pcie->pf_base + off);
+
+	return ioread32(pcie->pf_base + off);
+}
+
+static void ls_pcie_pf_writel(struct ls_pcie *pcie, u32 off, u32 val)
+{
+	if (pcie->big_endian)
+		return iowrite32be(val, pcie->pf_base + off);
+
+	return iowrite32(val, pcie->pf_base + off);
+
+}
+
+static void ls_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct ls_pcie *pcie = to_ls_pcie(pci);
+	u32 val;
+	int ret;
+
+	val = ls_pcie_pf_readl(pcie, LS_PCIE_PF_MCR);
+	val |= PF_MCR_PTOMR;
+	ls_pcie_pf_writel(pcie, LS_PCIE_PF_MCR, val);
+
+	ret = readx_poll_timeout(ls_pcie_pf_readl_addr, LS_PCIE_PF_MCR,
+				 val, !(val & PF_MCR_PTOMR), 100, 10000);
+	if (ret)
+		dev_info(pcie->pci->dev, "poll turn off message timeout\n");
+}
+
+static void ls_pcie_exit_from_l2(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct ls_pcie *pcie = to_ls_pcie(pci);
+	u32 val;
+	int ret;
+
+	val = ls_pcie_pf_readl(pcie, LS_PCIE_PF_MCR);
+	val |= PF_MCR_EXL2S;
+	ls_pcie_pf_writel(pcie, LS_PCIE_PF_MCR, val);
+
+	ret = readx_poll_timeout(ls_pcie_pf_readl_addr, LS_PCIE_PF_MCR,
+				 val, !(val & PF_MCR_EXL2S), 100, 10000);
+	if (ret)
+		dev_info(pcie->pci->dev, "poll exit L2 state timeout\n");
+}
+
 static int ls_pcie_host_init(struct dw_pcie_rp *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
@@ -91,18 +165,33 @@ static int ls_pcie_host_init(struct dw_pcie_rp *pp)
 
 static const struct dw_pcie_host_ops ls_pcie_host_ops = {
 	.host_init = ls_pcie_host_init,
+	.pme_turn_off = ls_pcie_send_turnoff_msg,
+	.exit_from_l2 = ls_pcie_exit_from_l2,
+};
+
+static const struct ls_pcie_drvdata ls1021a_drvdata = {
+};
+
+static const struct ls_pcie_drvdata ls1043a_drvdata = {
+	.lut_off = 0x10000,
+};
+
+static const struct ls_pcie_drvdata layerscape_drvdata = {
+	.lut_off = 0x80000,
+	.pf_off = 0xc0000,
+	.pm_support = true,
 };
 
 static const struct of_device_id ls_pcie_of_match[] = {
-	{ .compatible = "fsl,ls1012a-pcie", },
-	{ .compatible = "fsl,ls1021a-pcie", },
-	{ .compatible = "fsl,ls1028a-pcie", },
-	{ .compatible = "fsl,ls1043a-pcie", },
-	{ .compatible = "fsl,ls1046a-pcie", },
-	{ .compatible = "fsl,ls2080a-pcie", },
-	{ .compatible = "fsl,ls2085a-pcie", },
-	{ .compatible = "fsl,ls2088a-pcie", },
-	{ .compatible = "fsl,ls1088a-pcie", },
+	{ .compatible = "fsl,ls1012a-pcie", .data = &layerscape_drvdata },
+	{ .compatible = "fsl,ls1021a-pcie", .data = &ls1021a_drvdata },
+	{ .compatible = "fsl,ls1028a-pcie", .data = &layerscape_drvdata },
+	{ .compatible = "fsl,ls1043a-pcie", .data = &ls1043a_drvdata },
+	{ .compatible = "fsl,ls1046a-pcie", .data = &layerscape_drvdata },
+	{ .compatible = "fsl,ls2080a-pcie", .data = &layerscape_drvdata },
+	{ .compatible = "fsl,ls2085a-pcie", .data = &layerscape_drvdata },
+	{ .compatible = "fsl,ls2088a-pcie", .data = &layerscape_drvdata },
+	{ .compatible = "fsl,ls1088a-pcie", .data = &layerscape_drvdata },
 	{ },
 };
 
@@ -121,6 +210,8 @@ static int ls_pcie_probe(struct platform_device *pdev)
 	if (!pci)
 		return -ENOMEM;
 
+	pcie->drvdata = of_device_get_match_data(dev);
+
 	pci->dev = dev;
 	pci->pp.ops = &ls_pcie_host_ops;
 
@@ -131,6 +222,14 @@ static int ls_pcie_probe(struct platform_device *pdev)
 	if (IS_ERR(pci->dbi_base))
 		return PTR_ERR(pci->dbi_base);
 
+	pcie->big_endian = of_property_read_bool(dev->of_node, "big-endian");
+
+	if (pcie->drvdata->lut_off)
+		pcie->lut_base = pci->dbi_base + pcie->drvdata->lut_off;
+
+	if (pcie->drvdata->pf_off)
+		pcie->pf_base = pci->dbi_base + pcie->drvdata->pf_off;
+
 	if (!ls_pcie_is_bridge(pcie))
 		return -ENODEV;
 
@@ -139,12 +238,43 @@ static int ls_pcie_probe(struct platform_device *pdev)
 	return dw_pcie_host_init(&pci->pp);
 }
 
+#ifdef CONFIG_PM_SLEEP
+static int ls_pcie_suspend_noirq(struct device *dev)
+{
+	struct ls_pcie *pcie = dev_get_drvdata(dev);
+	struct dw_pcie *pci = pcie->pci;
+
+	if (!pcie->drvdata->pm_support)
+		return 0;
+
+	return dw_pcie_suspend_noirq(pci);
+}
+
+static int ls_pcie_resume_noirq(struct device *dev)
+{
+
+	struct ls_pcie *pcie = dev_get_drvdata(dev);
+	struct dw_pcie *pci = pcie->pci;
+
+	if (!pcie->drvdata->pm_support)
+		return 0;
+
+	return dw_pcie_resume_noirq(pci);
+}
+#endif /* CONFIG_PM_SLEEP */
+
+static const struct dev_pm_ops ls_pcie_pm_ops = {
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(ls_pcie_suspend_noirq,
+				      ls_pcie_resume_noirq)
+};
+
 static struct platform_driver ls_pcie_driver = {
 	.probe = ls_pcie_probe,
 	.driver = {
 		.name = "layerscape-pcie",
 		.of_match_table = ls_pcie_of_match,
 		.suppress_bind_attrs = true,
+		.pm = &ls_pcie_pm_ops,
 	},
 };
 builtin_platform_driver(ls_pcie_driver);
Signature · 1 line
2.34.1
#2 lore raw

Re: [PATCH v5 2/2] PCI: layerscape: Add power management support for ls1028a

Quoted 21 lines · Frank Li

From: Hou Zhiqiang <[email protected]>

Add PME_Turn_Off/PME_TO_Ack handshake sequence, and finally put the PCIe controller into D3 state after the L2/L3 ready state transition process completion.
...

drivers/pci/controller/dwc/pci-layerscape.c +14 -0
+#ifdef CONFIG_PM_SLEEP
+static int ls_pcie_suspend_noirq(struct device *dev)
+{
+	struct ls_pcie *pcie = dev_get_drvdata(dev);
+	struct dw_pcie *pci = pcie->pci;
+
+	if (!pcie->drvdata->pm_support)
+		return 0;
+
+	return dw_pcie_suspend_noirq(pci);
+}
+
+static int ls_pcie_resume_noirq(struct device *dev)
+{

drivers/pci/controller/dwc/pci-layerscape.c static int ls_pcie_probe(struct platform_device *pdev) · line 264

Quoted 10 lines
drivers/pci/controller/dwc/pci-layerscape.c +10 -0
+
+	struct ls_pcie *pcie = dev_get_drvdata(dev);
+	struct dw_pcie *pci = pcie->pci;
+
+	if (!pcie->drvdata->pm_support)
+		return 0;
+
+	return dw_pcie_resume_noirq(pci);
+}
+#endif /* CONFIG_PM_SLEEP */

You may be able to dispense with some of these #ifdefs as in
https://lore.kernel.org/all/20220720224829.GA1667002@bhelgaas/

Quoted 18 lines
drivers/pci/controller/dwc/pci-layerscape.c +6 -0
+static const struct dev_pm_ops ls_pcie_pm_ops = {
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(ls_pcie_suspend_noirq,
+				      ls_pcie_resume_noirq)
+};
+
 static struct platform_driver ls_pcie_driver = {
 	.probe = ls_pcie_probe,
 	.driver = {
 		.name = "layerscape-pcie",
 		.of_match_table = ls_pcie_of_match,
 		.suppress_bind_attrs = true,
+		.pm = &ls_pcie_pm_ops,
 	},
 };
 builtin_platform_driver(ls_pcie_driver);
Signature · 1 line
2.34.1
#3 lore raw
In reply to Frank Li
Quoted 2 lines · Frank Li

Introduced helper function dw_pcie_get_ltssm to retrieve SMLH_LTSS_STATE. Added API pme_turn_off and exit_from_l2 for managing L2/L3 state transitions.

Commit logs should not use the past tense but the present tense.

eg s/Introduced/Introduce

Furthermore, read this post from Bjorn and follow it to the letter please:

https://lore.kernel.org/linux-pci/[email protected]

Quoted 7 lines

Typical L2 entry workflow:

1. Transmit PME turn off signal to PCI devices and wait for PME_To_Ack.
2. Await link entering L2_IDLE state.
3. Transition Root complex to D3 state.

Quoted 6 lines

Typical L2 exit workflow:

1. Transition Root complex to D0 state.
2. Issue exit from L2 command.
3. Reinitialize PCI host.
4. Wait for link to become active.

This does not explain what the patch does and why it does it.

Are you describing the L2 entry/exit as implemented in the code ?

Quoted 7 lines

---
Change from v4 to v5:
- Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
Change from v3 to v4:
- change according to Manivannan's comments.

I shall wait for Mani's comments on this series since he reviewed the previous one.

Thanks,
Lorenzo

Quoted 193 lines

I hope I have not missed anything. quite long discuss thread
Change from v2 to v3:
- Basic rewrite whole patch according rob herry suggestion. put common function into dwc, so more soc can share the same logic.

 .../pci/controller/dwc/pcie-designware-host.c | 95 +++++++++++++++++++
 drivers/pci/controller/dwc/pcie-designware.h  | 28 ++++++
 2 files changed, 123 insertions(+)
drivers/pci/controller/dwc/pcie-designware-host.c +95 -0
index 9952057c8819..031e1f9c0d0c 100644
@@ -8,6 +8,7 @@
  * Author: Jingoo Han <[email protected]>
  */
 
+#include <linux/iopoll.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/irqdomain.h>
 #include <linux/msi.h>
@@ -807,3 +808,97 @@ int dw_pcie_setup_rc(struct dw_pcie_rp *pp)
 	return 0;
 }
 EXPORT_SYMBOL_GPL(dw_pcie_setup_rc);
+
+/*
+ * This resemble the pci_set_power_state() interfaces, but these are for
+ * configuring host controllers, which are bridges *to* PCI devices but
+ * are not PCI devices themselves.
+ */
+static void dw_pcie_set_dstate(struct dw_pcie *pci, pci_power_t dstate)
+{
+	u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_PM);
+	u16 val;
+
+	val = dw_pcie_readw_dbi(pci, offset + PCI_PM_CTRL);
+	val &= ~PCI_PM_CTRL_STATE_MASK;
+	val |= ((u16 __force)dstate) & PCI_PM_CTRL_STATE_MASK;
+	dw_pcie_writew_dbi(pci, offset + PCI_PM_CTRL, val);
+}
+
+int dw_pcie_suspend_noirq(struct dw_pcie *pci)
+{
+	u8 offset;
+	u32 val;
+	int ret;
+
+	offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
+	/*
+	 * If L1.1\L1.2 enable, devices (such as NVME) want short
+	 * resume latency, controller will not enter L2
+	 */
+	if (dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKCTL) & PCI_EXP_LNKCTL_ASPM_L1)
+		return 0;
+
+	if (dw_pcie_get_ltssm(pci) <= DW_PCIE_LTSSM_DETECT_ACT)
+		return 0;
+
+	if (!pci->pp.ops->pme_turn_off)
+		return -EINVAL;
+
+	pci->pp.ops->pme_turn_off(&pci->pp);
+
+	/*
+	 * PCI Express Base Specification Rev 4.0
+	 * 5.3.3.2.1 PME Synchronization
+	 * Recommand 1ms to 10ms timeout to check L2 ready
+	 */
+	ret = read_poll_timeout(dw_pcie_get_ltssm, val, val == DW_PCIE_LTSSM_L2_IDLE,
+				100, 10000, false, pci);
+	if (ret) {
+		dev_err(pci->dev, "PCIe link enter L2 timeout! ltssm = 0x%x\n", val);
+		return ret;
+	}
+
+	dw_pcie_set_dstate(pci, PCI_D3hot);
+
+	pci->suspended = true;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(dw_pcie_suspend_noirq);
+
+int dw_pcie_resume_noirq(struct dw_pcie *pci)
+{
+	int ret;
+
+	if (!pci->suspended)
+		return 0;
+
+	pci->suspended = false;
+
+	dw_pcie_set_dstate(pci, PCI_D0);
+
+	if (!pci->pp.ops->exit_from_l2)
+		return -EINVAL;
+
+	pci->pp.ops->exit_from_l2(&pci->pp);
+
+	ret = pci->pp.ops->host_init(&pci->pp);
+	if (ret) {
+		dev_err(pci->dev, "Host init failed! ret = 0x%x\n", ret);
+		return ret;
+	}
+
+	dw_pcie_setup_rc(&pci->pp);
+
+	ret = dw_pcie_start_link(pci);
+	if (ret)
+		return ret;
+
+	ret = dw_pcie_wait_for_link(pci);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(dw_pcie_resume_noirq);
drivers/pci/controller/dwc/pcie-designware.h +28 -0
index 79713ce075cc..effb07a506e4 100644
@@ -288,10 +288,21 @@ enum dw_pcie_core_rst {
 	DW_PCIE_NUM_CORE_RSTS
 };
 
+enum dw_pcie_ltssm {
+	DW_PCIE_LTSSM_UNKNOWN = 0xFFFFFFFF,
+	/* Need align PCIE_PORT_DEBUG0 bit0:5 */
+	DW_PCIE_LTSSM_DETECT_QUIET = 0x0,
+	DW_PCIE_LTSSM_DETECT_ACT = 0x1,
+	DW_PCIE_LTSSM_L0 = 0x11,
+	DW_PCIE_LTSSM_L2_IDLE = 0x15,
+};
+
 struct dw_pcie_host_ops {
 	int (*host_init)(struct dw_pcie_rp *pp);
 	void (*host_deinit)(struct dw_pcie_rp *pp);
 	int (*msi_host_init)(struct dw_pcie_rp *pp);
+	void (*pme_turn_off)(struct dw_pcie_rp *pp);
+	void (*exit_from_l2)(struct dw_pcie_rp *pp);
 };
 
 struct dw_pcie_rp {
@@ -364,6 +375,7 @@ struct dw_pcie_ops {
 	void    (*write_dbi2)(struct dw_pcie *pcie, void __iomem *base, u32 reg,
 			      size_t size, u32 val);
 	int	(*link_up)(struct dw_pcie *pcie);
+	enum dw_pcie_ltssm (*get_ltssm)(struct dw_pcie *pcie);
 	int	(*start_link)(struct dw_pcie *pcie);
 	void	(*stop_link)(struct dw_pcie *pcie);
 };
@@ -393,6 +405,7 @@ struct dw_pcie {
 	struct reset_control_bulk_data	app_rsts[DW_PCIE_NUM_APP_RSTS];
 	struct reset_control_bulk_data	core_rsts[DW_PCIE_NUM_CORE_RSTS];
 	struct gpio_desc		*pe_rst;
+	bool			suspended;
 };
 
 #define to_dw_pcie_from_pp(port) container_of((port), struct dw_pcie, pp)
@@ -430,6 +443,9 @@ void dw_pcie_iatu_detect(struct dw_pcie *pci);
 int dw_pcie_edma_detect(struct dw_pcie *pci);
 void dw_pcie_edma_remove(struct dw_pcie *pci);
 
+int dw_pcie_suspend_noirq(struct dw_pcie *pci);
+int dw_pcie_resume_noirq(struct dw_pcie *pci);
+
 static inline void dw_pcie_writel_dbi(struct dw_pcie *pci, u32 reg, u32 val)
 {
 	dw_pcie_write_dbi(pci, reg, 0x4, val);
@@ -501,6 +517,18 @@ static inline void dw_pcie_stop_link(struct dw_pcie *pci)
 		pci->ops->stop_link(pci);
 }
 
+static inline enum dw_pcie_ltssm dw_pcie_get_ltssm(struct dw_pcie *pci)
+{
+	u32 val;
+
+	if (pci->ops && pci->ops->get_ltssm)
+		return pci->ops->get_ltssm(pci);
+
+	val = dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0);
+
+	return (enum dw_pcie_ltssm)FIELD_GET(PORT_LOGIC_LTSSM_STATE_MASK, val);
+}
+
 #ifdef CONFIG_PCIE_DW_HOST
 irqreturn_t dw_handle_msi_irq(struct dw_pcie_rp *pp);
 int dw_pcie_setup_rc(struct dw_pcie_rp *pp);
Signature · 1 line
2.34.1
author maintainer
#4 lore raw
Quoted 22 lines · Lorenzo Pieralisi
Quoted 2 lines · Frank Li

Introduced helper function dw_pcie_get_ltssm to retrieve SMLH_LTSS_STATE. Added API pme_turn_off and exit_from_l2 for managing L2/L3 state transitions.

Commit logs should not use the past tense but the present tense.

eg s/Introduced/Introduce

Furthermore, read this post from Bjorn and follow it to the letter please:

https://lore.kernel.org/linux-pci/[email protected]

Quoted 9 lines

Typical L2 entry workflow:

1. Transmit PME turn off signal to PCI devices and wait for PME_To_Ack.
2. Await link entering L2_IDLE state.
3. Transition Root complex to D3 state.

Typical L2 exit workflow:

Quoted 8 lines
Quoted 4 lines

1. Transition Root complex to D0 state.
2. Issue exit from L2 command.
3. Reinitialize PCI host.
4. Wait for link to become active.

This does not explain what the patch does and why it does it.

Are you describing the L2 entry/exit as implemented in the code ?

Yes.

How about below commit message:

Introduce helper function dw_pcie_get_ltssm to retrieve SMLH_LTSS_STATE.

Add callback .pme_turn_off and .exit_from_l2 for platform specific PME handle.

Add common dw_pcie_suspend(resume)_noirq() API to avoid duplicated code in dwc pci host controller platform driver.

Typical L2 entry workflow/dw_pcie_suspend_noirq()

1. Transmit PME turn off signal to PCI devices and wait for PME_To_Ack.
2. Await link entering L2_IDLE state.
3. Transition Root complex to D3 state.

Typical L2 exit workflow/dw_pcie_resume_noirq()

1. Transition Root complex to D0 state.
2. Issue exit from L2 command.
3. Reinitialize PCI host.
4. Wait for link to become active.

Frank

Quoted 5 lines
Quoted 5 lines

---
Change from v4 to v5:

#5 lore raw
In reply to Frank Li
Quoted 44 lines · Frank Li

Introduced helper function dw_pcie_get_ltssm to retrieve SMLH_LTSS_STATE. Added API pme_turn_off and exit_from_l2 for managing L2/L3 state transitions.

Typical L2 entry workflow:

1. Transmit PME turn off signal to PCI devices and wait for PME_To_Ack.
2. Await link entering L2_IDLE state.
3. Transition Root complex to D3 state.

Typical L2 exit workflow:

1. Transition Root complex to D0 state.
2. Issue exit from L2 command.
3. Reinitialize PCI host.
4. Wait for link to become active.

---
Change from v4 to v5:
- Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
Change from v3 to v4:
- change according to Manivannan's comments.
I hope I have not missed anything. quite long discuss thread
Change from v2 to v3:
- Basic rewrite whole patch according rob herry suggestion.
put common function into dwc, so more soc can share the same logic.

 .../pci/controller/dwc/pcie-designware-host.c | 95 +++++++++++++++++++
 drivers/pci/controller/dwc/pcie-designware.h  | 28 ++++++
 2 files changed, 123 insertions(+)
drivers/pci/controller/dwc/pcie-designware-host.c +1 -0
index 9952057c8819..031e1f9c0d0c 100644
@@ -8,6 +8,7 @@
  * Author: Jingoo Han <[email protected]>
  */
 
+#include <linux/iopoll.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/irqdomain.h>
 #include <linux/msi.h>

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 813

Quoted 6 lines
drivers/pci/controller/dwc/pcie-designware-host.c +3 -0
@@ -807,3 +808,97 @@ int dw_pcie_setup_rc(struct dw_pcie_rp *pp)
 	return 0;
 }
 EXPORT_SYMBOL_GPL(dw_pcie_setup_rc);
+
+/*
+ * This resemble the pci_set_power_state() interfaces, but these are for

s/interfaces/API
s/these are/this is

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 824

Quoted 11 lines
drivers/pci/controller/dwc/pcie-designware-host.c +11 -0
+ * configuring host controllers, which are bridges *to* PCI devices but
+ * are not PCI devices themselves.
+ */
+static void dw_pcie_set_dstate(struct dw_pcie *pci, pci_power_t dstate)
+{
+	u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_PM);
+	u16 val;
+
+	val = dw_pcie_readw_dbi(pci, offset + PCI_PM_CTRL);
+	val &= ~PCI_PM_CTRL_STATE_MASK;
+	val |= ((u16 __force)dstate) & PCI_PM_CTRL_STATE_MASK;

Why can't just,

val |= dstate;

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 834

Quoted 10 lines
drivers/pci/controller/dwc/pcie-designware-host.c +10 -0
+	dw_pcie_writew_dbi(pci, offset + PCI_PM_CTRL, val);
+}
+
+int dw_pcie_suspend_noirq(struct dw_pcie *pci)
+{
+	u8 offset;
+	u32 val;
+	int ret;
+
+	offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);

Just assign this during variable definition itself..

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 837

Quoted 3 lines
drivers/pci/controller/dwc/pcie-designware-host.c +3 -0
+	/*
+	 * If L1.1\L1.2 enable, devices (such as NVME) want short
+	 * resume latency, controller will not enter L2

"If L1SS is supported, then do not put the link into L2 as some devices such as NVMe expect low resume latency."

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 846

Quoted 9 lines
drivers/pci/controller/dwc/pcie-designware-host.c +9 -0
+	 */
+	if (dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKCTL) & PCI_EXP_LNKCTL_ASPM_L1)
+		return 0;
+
+	if (dw_pcie_get_ltssm(pci) <= DW_PCIE_LTSSM_DETECT_ACT)
+		return 0;
+
+	if (!pci->pp.ops->pme_turn_off)
+		return -EINVAL;

Can you just return 0 instead of failing? As per my comment below, if you move the D-state change before this callback, then this won't be a hard requirement.

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 853

Quoted 7 lines
drivers/pci/controller/dwc/pcie-designware-host.c +7 -0
+
+	pci->pp.ops->pme_turn_off(&pci->pp);
+
+	/*
+	 * PCI Express Base Specification Rev 4.0
+	 * 5.3.3.2.1 PME Synchronization
+	 * Recommand 1ms to 10ms timeout to check L2 ready

"Recommends"

Please use full 80 columns for comments.

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 858

Quoted 5 lines
drivers/pci/controller/dwc/pcie-designware-host.c +5 -0
+	 */
+	ret = read_poll_timeout(dw_pcie_get_ltssm, val, val == DW_PCIE_LTSSM_L2_IDLE,
+				100, 10000, false, pci);
+	if (ret) {
+		dev_err(pci->dev, "PCIe link enter L2 timeout! ltssm = 0x%x\n", val);

"Timeout waiting for L2 entry! LTSSM: 0x%x\n"

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 862

Quoted 4 lines
drivers/pci/controller/dwc/pcie-designware-host.c +4 -0
+		return ret;
+	}
+
+	dw_pcie_set_dstate(pci, PCI_D3hot);

This should be done before initiating L2 entry as per PCI_Express_Base_Specification 3.0, section 5.2.

Quoted 10 lines
drivers/pci/controller/dwc/pcie-designware-host.c +10 -0
+
+	pci->suspended = true;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(dw_pcie_suspend_noirq);
+
+int dw_pcie_resume_noirq(struct dw_pcie *pci)
+{
+	int ret;

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 882

Quoted 10 lines
drivers/pci/controller/dwc/pcie-designware-host.c +10 -0
+
+	if (!pci->suspended)
+		return 0;
+
+	pci->suspended = false;
+
+	dw_pcie_set_dstate(pci, PCI_D0);
+
+	if (!pci->pp.ops->exit_from_l2)
+		return -EINVAL;

Same comment as above.

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 886

Quoted 4 lines
drivers/pci/controller/dwc/pcie-designware-host.c +4 -0
+
+	pci->pp.ops->exit_from_l2(&pci->pp);
+
+	ret = pci->pp.ops->host_init(&pci->pp);

I don't see matching host_deinit() in suspend.

- Mani

Quoted 23 lines
drivers/pci/controller/dwc/pcie-designware-host.c +18 -0
+	if (ret) {
+		dev_err(pci->dev, "Host init failed! ret = 0x%x\n", ret);
+		return ret;
+	}
+
+	dw_pcie_setup_rc(&pci->pp);
+
+	ret = dw_pcie_start_link(pci);
+	if (ret)
+		return ret;
+
+	ret = dw_pcie_wait_for_link(pci);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(dw_pcie_resume_noirq);
drivers/pci/controller/dwc/pcie-designware.h +0 -0
index 79713ce075cc..effb07a506e4 100644

drivers/pci/controller/dwc/pcie-designware.h enum dw_pcie_core_rst { · line 293

Quoted 6 lines
drivers/pci/controller/dwc/pcie-designware.h +3 -0
@@ -288,10 +288,21 @@ enum dw_pcie_core_rst {
 	DW_PCIE_NUM_CORE_RSTS
 };
 
+enum dw_pcie_ltssm {
+	DW_PCIE_LTSSM_UNKNOWN = 0xFFFFFFFF,
+	/* Need align PCIE_PORT_DEBUG0 bit0:5 */

"Need to align with PCIE_PORT_DEBUG0 bits 0:5".

- Mani

Quoted 63 lines
drivers/pci/controller/dwc/pcie-designware.h +25 -0
+	DW_PCIE_LTSSM_DETECT_QUIET = 0x0,
+	DW_PCIE_LTSSM_DETECT_ACT = 0x1,
+	DW_PCIE_LTSSM_L0 = 0x11,
+	DW_PCIE_LTSSM_L2_IDLE = 0x15,
+};
+
 struct dw_pcie_host_ops {
 	int (*host_init)(struct dw_pcie_rp *pp);
 	void (*host_deinit)(struct dw_pcie_rp *pp);
 	int (*msi_host_init)(struct dw_pcie_rp *pp);
+	void (*pme_turn_off)(struct dw_pcie_rp *pp);
+	void (*exit_from_l2)(struct dw_pcie_rp *pp);
 };
 
 struct dw_pcie_rp {
@@ -364,6 +375,7 @@ struct dw_pcie_ops {
 	void    (*write_dbi2)(struct dw_pcie *pcie, void __iomem *base, u32 reg,
 			      size_t size, u32 val);
 	int	(*link_up)(struct dw_pcie *pcie);
+	enum dw_pcie_ltssm (*get_ltssm)(struct dw_pcie *pcie);
 	int	(*start_link)(struct dw_pcie *pcie);
 	void	(*stop_link)(struct dw_pcie *pcie);
 };
@@ -393,6 +405,7 @@ struct dw_pcie {
 	struct reset_control_bulk_data	app_rsts[DW_PCIE_NUM_APP_RSTS];
 	struct reset_control_bulk_data	core_rsts[DW_PCIE_NUM_CORE_RSTS];
 	struct gpio_desc		*pe_rst;
+	bool			suspended;
 };
 
 #define to_dw_pcie_from_pp(port) container_of((port), struct dw_pcie, pp)
@@ -430,6 +443,9 @@ void dw_pcie_iatu_detect(struct dw_pcie *pci);
 int dw_pcie_edma_detect(struct dw_pcie *pci);
 void dw_pcie_edma_remove(struct dw_pcie *pci);
 
+int dw_pcie_suspend_noirq(struct dw_pcie *pci);
+int dw_pcie_resume_noirq(struct dw_pcie *pci);
+
 static inline void dw_pcie_writel_dbi(struct dw_pcie *pci, u32 reg, u32 val)
 {
 	dw_pcie_write_dbi(pci, reg, 0x4, val);
@@ -501,6 +517,18 @@ static inline void dw_pcie_stop_link(struct dw_pcie *pci)
 		pci->ops->stop_link(pci);
 }
 
+static inline enum dw_pcie_ltssm dw_pcie_get_ltssm(struct dw_pcie *pci)
+{
+	u32 val;
+
+	if (pci->ops && pci->ops->get_ltssm)
+		return pci->ops->get_ltssm(pci);
+
+	val = dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0);
+
+	return (enum dw_pcie_ltssm)FIELD_GET(PORT_LOGIC_LTSSM_STATE_MASK, val);
+}
+
 #ifdef CONFIG_PCIE_DW_HOST
 irqreturn_t dw_handle_msi_irq(struct dw_pcie_rp *pp);
 int dw_pcie_setup_rc(struct dw_pcie_rp *pp);
Signature · 1 line
2.34.1
Signature · 1 line
மணிவண்ணன் சதாசிவம்
author maintainer
#6 lore raw

Re: [PATCH v5 1/2] PCI: dwc: Implement general suspend/resume functionality for L2/L3 transitionse

Quoted 69 lines · Manivannan Sadhasivam
Quoted 44 lines · Frank Li

Introduced helper function dw_pcie_get_ltssm to retrieve SMLH_LTSS_STATE. Added API pme_turn_off and exit_from_l2 for managing L2/L3 state transitions.

Typical L2 entry workflow:

1. Transmit PME turn off signal to PCI devices and wait for PME_To_Ack.
2. Await link entering L2_IDLE state.
3. Transition Root complex to D3 state.

Typical L2 exit workflow:

1. Transition Root complex to D0 state.
2. Issue exit from L2 command.
3. Reinitialize PCI host.
4. Wait for link to become active.

---
Change from v4 to v5:
- Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
Change from v3 to v4:
- change according to Manivannan's comments.
I hope I have not missed anything. quite long discuss thread
Change from v2 to v3:
- Basic rewrite whole patch according rob herry suggestion.
put common function into dwc, so more soc can share the same logic.

 .../pci/controller/dwc/pcie-designware-host.c | 95 +++++++++++++++++++
 drivers/pci/controller/dwc/pcie-designware.h  | 28 ++++++
 2 files changed, 123 insertions(+)
drivers/pci/controller/dwc/pcie-designware-host.c +1 -0
index 9952057c8819..031e1f9c0d0c 100644
@@ -8,6 +8,7 @@
  * Author: Jingoo Han <[email protected]>
  */
 
+#include <linux/iopoll.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/irqdomain.h>
 #include <linux/msi.h>

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 813

Quoted 6 lines
drivers/pci/controller/dwc/pcie-designware-host.c +3 -0
@@ -807,3 +808,97 @@ int dw_pcie_setup_rc(struct dw_pcie_rp *pp)
 	return 0;
 }
 EXPORT_SYMBOL_GPL(dw_pcie_setup_rc);
+
+/*
+ * This resemble the pci_set_power_state() interfaces, but these are for

s/interfaces/API
s/these are/this is

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 824

Quoted 11 lines
drivers/pci/controller/dwc/pcie-designware-host.c +11 -0
+ * configuring host controllers, which are bridges *to* PCI devices but
+ * are not PCI devices themselves.
+ */
+static void dw_pcie_set_dstate(struct dw_pcie *pci, pci_power_t dstate)
+{
+	u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_PM);
+	u16 val;
+
+	val = dw_pcie_readw_dbi(pci, offset + PCI_PM_CTRL);
+	val &= ~PCI_PM_CTRL_STATE_MASK;
+	val |= ((u16 __force)dstate) & PCI_PM_CTRL_STATE_MASK;

Why can't just,

Quoted 1 line

val |= dstate;

fixed a build warning.

Quoted 13 lines

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 834

Quoted 11 lines
drivers/pci/controller/dwc/pcie-designware-host.c +10 -0
+	dw_pcie_writew_dbi(pci, offset + PCI_PM_CTRL, val);
+}
+
+int dw_pcie_suspend_noirq(struct dw_pcie *pci)
+{
+	u8 offset;
+	u32 val;
+	int ret;
+
+	offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);

Just assign this during variable definition itself..

Do you means?

	u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
Quoted 41 lines

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 837

Quoted 4 lines
drivers/pci/controller/dwc/pcie-designware-host.c +3 -0
+	/*
+	 * If L1.1\L1.2 enable, devices (such as NVME) want short
+	 * resume latency, controller will not enter L2

"If L1SS is supported, then do not put the link into L2 as some devices such as NVMe expect low resume latency."

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 846

Quoted 9 lines
drivers/pci/controller/dwc/pcie-designware-host.c +9 -0
+	 */
+	if (dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKCTL) & PCI_EXP_LNKCTL_ASPM_L1)
+		return 0;
+
+	if (dw_pcie_get_ltssm(pci) <= DW_PCIE_LTSSM_DETECT_ACT)
+		return 0;
+
+	if (!pci->pp.ops->pme_turn_off)
+		return -EINVAL;

Can you just return 0 instead of failing? As per my comment below, if you move the D-state change before this callback, then this won't be a hard requirement.

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 853

Quoted 7 lines
drivers/pci/controller/dwc/pcie-designware-host.c +7 -0
+
+	pci->pp.ops->pme_turn_off(&pci->pp);
+
+	/*
+	 * PCI Express Base Specification Rev 4.0
+	 * 5.3.3.2.1 PME Synchronization
+	 * Recommand 1ms to 10ms timeout to check L2 ready

"Recommends"

Please use full 80 columns for comments.

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 858

Quoted 5 lines
drivers/pci/controller/dwc/pcie-designware-host.c +5 -0
+	 */
+	ret = read_poll_timeout(dw_pcie_get_ltssm, val, val == DW_PCIE_LTSSM_L2_IDLE,
+				100, 10000, false, pci);
+	if (ret) {
+		dev_err(pci->dev, "PCIe link enter L2 timeout! ltssm = 0x%x\n", val);

"Timeout waiting for L2 entry! LTSSM: 0x%x\n"

Quoted 7 lines

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 862

Quoted 4 lines
drivers/pci/controller/dwc/pcie-designware-host.c +4 -0
+		return ret;
+	}
+
+	dw_pcie_set_dstate(pci, PCI_D3hot);

This should be done before initiating L2 entry as per PCI_Express_Base_Specification 3.0, section 5.2.

This is for root complex bridge, after PCI device and link to L2, PCI Host can be enter lower power state, I think it is just borrow 'D3 state' from PCI.

Quoted 133 lines
Quoted 11 lines
drivers/pci/controller/dwc/pcie-designware-host.c +10 -0
+
+	pci->suspended = true;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(dw_pcie_suspend_noirq);
+
+int dw_pcie_resume_noirq(struct dw_pcie *pci)
+{
+	int ret;

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 882

Quoted 10 lines
drivers/pci/controller/dwc/pcie-designware-host.c +10 -0
+
+	if (!pci->suspended)
+		return 0;
+
+	pci->suspended = false;
+
+	dw_pcie_set_dstate(pci, PCI_D0);
+
+	if (!pci->pp.ops->exit_from_l2)
+		return -EINVAL;

Same comment as above.

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 886

Quoted 4 lines
drivers/pci/controller/dwc/pcie-designware-host.c +4 -0
+
+	pci->pp.ops->exit_from_l2(&pci->pp);
+
+	ret = pci->pp.ops->host_init(&pci->pp);

I don't see matching host_deinit() in suspend.

- Mani

Quoted 23 lines
drivers/pci/controller/dwc/pcie-designware-host.c +18 -0
+	if (ret) {
+		dev_err(pci->dev, "Host init failed! ret = 0x%x\n", ret);
+		return ret;
+	}
+
+	dw_pcie_setup_rc(&pci->pp);
+
+	ret = dw_pcie_start_link(pci);
+	if (ret)
+		return ret;
+
+	ret = dw_pcie_wait_for_link(pci);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(dw_pcie_resume_noirq);
drivers/pci/controller/dwc/pcie-designware.h +0 -0
index 79713ce075cc..effb07a506e4 100644

drivers/pci/controller/dwc/pcie-designware.h enum dw_pcie_core_rst { · line 293

Quoted 6 lines
drivers/pci/controller/dwc/pcie-designware.h +3 -0
@@ -288,10 +288,21 @@ enum dw_pcie_core_rst {
 	DW_PCIE_NUM_CORE_RSTS
 };
 
+enum dw_pcie_ltssm {
+	DW_PCIE_LTSSM_UNKNOWN = 0xFFFFFFFF,
+	/* Need align PCIE_PORT_DEBUG0 bit0:5 */

"Need to align with PCIE_PORT_DEBUG0 bits 0:5".

- Mani

Quoted 63 lines
drivers/pci/controller/dwc/pcie-designware.h +25 -0
+	DW_PCIE_LTSSM_DETECT_QUIET = 0x0,
+	DW_PCIE_LTSSM_DETECT_ACT = 0x1,
+	DW_PCIE_LTSSM_L0 = 0x11,
+	DW_PCIE_LTSSM_L2_IDLE = 0x15,
+};
+
 struct dw_pcie_host_ops {
 	int (*host_init)(struct dw_pcie_rp *pp);
 	void (*host_deinit)(struct dw_pcie_rp *pp);
 	int (*msi_host_init)(struct dw_pcie_rp *pp);
+	void (*pme_turn_off)(struct dw_pcie_rp *pp);
+	void (*exit_from_l2)(struct dw_pcie_rp *pp);
 };
 
 struct dw_pcie_rp {
@@ -364,6 +375,7 @@ struct dw_pcie_ops {
 	void    (*write_dbi2)(struct dw_pcie *pcie, void __iomem *base, u32 reg,
 			      size_t size, u32 val);
 	int	(*link_up)(struct dw_pcie *pcie);
+	enum dw_pcie_ltssm (*get_ltssm)(struct dw_pcie *pcie);
 	int	(*start_link)(struct dw_pcie *pcie);
 	void	(*stop_link)(struct dw_pcie *pcie);
 };
@@ -393,6 +405,7 @@ struct dw_pcie {
 	struct reset_control_bulk_data	app_rsts[DW_PCIE_NUM_APP_RSTS];
 	struct reset_control_bulk_data	core_rsts[DW_PCIE_NUM_CORE_RSTS];
 	struct gpio_desc		*pe_rst;
+	bool			suspended;
 };
 
 #define to_dw_pcie_from_pp(port) container_of((port), struct dw_pcie, pp)
@@ -430,6 +443,9 @@ void dw_pcie_iatu_detect(struct dw_pcie *pci);
 int dw_pcie_edma_detect(struct dw_pcie *pci);
 void dw_pcie_edma_remove(struct dw_pcie *pci);
 
+int dw_pcie_suspend_noirq(struct dw_pcie *pci);
+int dw_pcie_resume_noirq(struct dw_pcie *pci);
+
 static inline void dw_pcie_writel_dbi(struct dw_pcie *pci, u32 reg, u32 val)
 {
 	dw_pcie_write_dbi(pci, reg, 0x4, val);
@@ -501,6 +517,18 @@ static inline void dw_pcie_stop_link(struct dw_pcie *pci)
 		pci->ops->stop_link(pci);
 }
 
+static inline enum dw_pcie_ltssm dw_pcie_get_ltssm(struct dw_pcie *pci)
+{
+	u32 val;
+
+	if (pci->ops && pci->ops->get_ltssm)
+		return pci->ops->get_ltssm(pci);
+
+	val = dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0);
+
+	return (enum dw_pcie_ltssm)FIELD_GET(PORT_LOGIC_LTSSM_STATE_MASK, val);
+}
+
 #ifdef CONFIG_PCIE_DW_HOST
 irqreturn_t dw_handle_msi_irq(struct dw_pcie_rp *pp);
 int dw_pcie_setup_rc(struct dw_pcie_rp *pp);
Signature · 1 line
2.34.1
Signature · 1 line
மணிவண்ணன் சதாசிவம்
#7 lore raw

Re: [PATCH v5 1/2] PCI: dwc: Implement general suspend/resume functionality for L2/L3 transitionse

Quoted 70 lines · Frank Li
Quoted 69 lines · Manivannan Sadhasivam
Quoted 44 lines · Frank Li

Introduced helper function dw_pcie_get_ltssm to retrieve SMLH_LTSS_STATE. Added API pme_turn_off and exit_from_l2 for managing L2/L3 state transitions.

Typical L2 entry workflow:

1. Transmit PME turn off signal to PCI devices and wait for PME_To_Ack.
2. Await link entering L2_IDLE state.
3. Transition Root complex to D3 state.

Typical L2 exit workflow:

1. Transition Root complex to D0 state.
2. Issue exit from L2 command.
3. Reinitialize PCI host.
4. Wait for link to become active.

---
Change from v4 to v5:
- Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
Change from v3 to v4:
- change according to Manivannan's comments.
I hope I have not missed anything. quite long discuss thread
Change from v2 to v3:
- Basic rewrite whole patch according rob herry suggestion.
put common function into dwc, so more soc can share the same logic.

 .../pci/controller/dwc/pcie-designware-host.c | 95 +++++++++++++++++++
 drivers/pci/controller/dwc/pcie-designware.h  | 28 ++++++
 2 files changed, 123 insertions(+)
drivers/pci/controller/dwc/pcie-designware-host.c +1 -0
index 9952057c8819..031e1f9c0d0c 100644
@@ -8,6 +8,7 @@
  * Author: Jingoo Han <[email protected]>
  */
 
+#include <linux/iopoll.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/irqdomain.h>
 #include <linux/msi.h>

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 813

Quoted 6 lines
drivers/pci/controller/dwc/pcie-designware-host.c +3 -0
@@ -807,3 +808,97 @@ int dw_pcie_setup_rc(struct dw_pcie_rp *pp)
 	return 0;
 }
 EXPORT_SYMBOL_GPL(dw_pcie_setup_rc);
+
+/*
+ * This resemble the pci_set_power_state() interfaces, but these are for

s/interfaces/API
s/these are/this is

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 824

Quoted 11 lines
drivers/pci/controller/dwc/pcie-designware-host.c +11 -0
+ * configuring host controllers, which are bridges *to* PCI devices but
+ * are not PCI devices themselves.
+ */
+static void dw_pcie_set_dstate(struct dw_pcie *pci, pci_power_t dstate)
+{
+	u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_PM);
+	u16 val;
+
+	val = dw_pcie_readw_dbi(pci, offset + PCI_PM_CTRL);
+	val &= ~PCI_PM_CTRL_STATE_MASK;
+	val |= ((u16 __force)dstate) & PCI_PM_CTRL_STATE_MASK;

Why can't just,

Quoted 6 lines
Quoted 1 line

val |= dstate;

fixed a build warning.

Ah ok. Missed this report. But still you can get rid of the mask part.

Quoted 14 lines
Quoted 13 lines

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 834

Quoted 11 lines
drivers/pci/controller/dwc/pcie-designware-host.c +10 -0
+	dw_pcie_writew_dbi(pci, offset + PCI_PM_CTRL, val);
+}
+
+int dw_pcie_suspend_noirq(struct dw_pcie *pci)
+{
+	u8 offset;
+	u32 val;
+	int ret;
+
+	offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);

Just assign this during variable definition itself..

Quoted 2 lines

Do you means?

	u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);

Yes!

Quoted 49 lines
Quoted 41 lines

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 837

Quoted 4 lines
drivers/pci/controller/dwc/pcie-designware-host.c +3 -0
+	/*
+	 * If L1.1\L1.2 enable, devices (such as NVME) want short
+	 * resume latency, controller will not enter L2

"If L1SS is supported, then do not put the link into L2 as some devices such as NVMe expect low resume latency."

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 846

Quoted 9 lines
drivers/pci/controller/dwc/pcie-designware-host.c +9 -0
+	 */
+	if (dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKCTL) & PCI_EXP_LNKCTL_ASPM_L1)
+		return 0;
+
+	if (dw_pcie_get_ltssm(pci) <= DW_PCIE_LTSSM_DETECT_ACT)
+		return 0;
+
+	if (!pci->pp.ops->pme_turn_off)
+		return -EINVAL;

Can you just return 0 instead of failing? As per my comment below, if you move the D-state change before this callback, then this won't be a hard requirement.

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 853

Quoted 7 lines
drivers/pci/controller/dwc/pcie-designware-host.c +7 -0
+
+	pci->pp.ops->pme_turn_off(&pci->pp);
+
+	/*
+	 * PCI Express Base Specification Rev 4.0
+	 * 5.3.3.2.1 PME Synchronization
+	 * Recommand 1ms to 10ms timeout to check L2 ready

"Recommends"

Please use full 80 columns for comments.

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 858

Quoted 5 lines
drivers/pci/controller/dwc/pcie-designware-host.c +5 -0
+	 */
+	ret = read_poll_timeout(dw_pcie_get_ltssm, val, val == DW_PCIE_LTSSM_L2_IDLE,
+				100, 10000, false, pci);
+	if (ret) {
+		dev_err(pci->dev, "PCIe link enter L2 timeout! ltssm = 0x%x\n", val);

"Timeout waiting for L2 entry! LTSSM: 0x%x\n"

Quoted 7 lines

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 862

Quoted 4 lines
drivers/pci/controller/dwc/pcie-designware-host.c +4 -0
+		return ret;
+	}
+
+	dw_pcie_set_dstate(pci, PCI_D3hot);

This should be done before initiating L2 entry as per PCI_Express_Base_Specification 3.0, section 5.2.

Quoted 4 lines

This is for root complex bridge, after PCI device and link to L2, PCI Host can be enter lower power state, I think it is just borrow 'D3 state' from PCI.

This is confusing now. What is the behavior if the root complex changes D-state but the device driver expects a different one? For instance, during suspend, the NVMe driver expects the D-state to be D0 only [1] not to be changed by the PCI core. This might apply to other drivers as well.

Better to leave the D-state change to PCI core itself.

- Mani

[1] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/nvme/host/pci.c#n3218

Quoted 133 lines
Quoted 133 lines
Quoted 11 lines
drivers/pci/controller/dwc/pcie-designware-host.c +10 -0
+
+	pci->suspended = true;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(dw_pcie_suspend_noirq);
+
+int dw_pcie_resume_noirq(struct dw_pcie *pci)
+{
+	int ret;

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 882

Quoted 10 lines
drivers/pci/controller/dwc/pcie-designware-host.c +10 -0
+
+	if (!pci->suspended)
+		return 0;
+
+	pci->suspended = false;
+
+	dw_pcie_set_dstate(pci, PCI_D0);
+
+	if (!pci->pp.ops->exit_from_l2)
+		return -EINVAL;

Same comment as above.

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 886

Quoted 4 lines
drivers/pci/controller/dwc/pcie-designware-host.c +4 -0
+
+	pci->pp.ops->exit_from_l2(&pci->pp);
+
+	ret = pci->pp.ops->host_init(&pci->pp);

I don't see matching host_deinit() in suspend.

- Mani

Quoted 23 lines
drivers/pci/controller/dwc/pcie-designware-host.c +18 -0
+	if (ret) {
+		dev_err(pci->dev, "Host init failed! ret = 0x%x\n", ret);
+		return ret;
+	}
+
+	dw_pcie_setup_rc(&pci->pp);
+
+	ret = dw_pcie_start_link(pci);
+	if (ret)
+		return ret;
+
+	ret = dw_pcie_wait_for_link(pci);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(dw_pcie_resume_noirq);
drivers/pci/controller/dwc/pcie-designware.h +0 -0
index 79713ce075cc..effb07a506e4 100644

drivers/pci/controller/dwc/pcie-designware.h enum dw_pcie_core_rst { · line 293

Quoted 6 lines
drivers/pci/controller/dwc/pcie-designware.h +3 -0
@@ -288,10 +288,21 @@ enum dw_pcie_core_rst {
 	DW_PCIE_NUM_CORE_RSTS
 };
 
+enum dw_pcie_ltssm {
+	DW_PCIE_LTSSM_UNKNOWN = 0xFFFFFFFF,
+	/* Need align PCIE_PORT_DEBUG0 bit0:5 */

"Need to align with PCIE_PORT_DEBUG0 bits 0:5".

- Mani

Quoted 63 lines
drivers/pci/controller/dwc/pcie-designware.h +25 -0
+	DW_PCIE_LTSSM_DETECT_QUIET = 0x0,
+	DW_PCIE_LTSSM_DETECT_ACT = 0x1,
+	DW_PCIE_LTSSM_L0 = 0x11,
+	DW_PCIE_LTSSM_L2_IDLE = 0x15,
+};
+
 struct dw_pcie_host_ops {
 	int (*host_init)(struct dw_pcie_rp *pp);
 	void (*host_deinit)(struct dw_pcie_rp *pp);
 	int (*msi_host_init)(struct dw_pcie_rp *pp);
+	void (*pme_turn_off)(struct dw_pcie_rp *pp);
+	void (*exit_from_l2)(struct dw_pcie_rp *pp);
 };
 
 struct dw_pcie_rp {
@@ -364,6 +375,7 @@ struct dw_pcie_ops {
 	void    (*write_dbi2)(struct dw_pcie *pcie, void __iomem *base, u32 reg,
 			      size_t size, u32 val);
 	int	(*link_up)(struct dw_pcie *pcie);
+	enum dw_pcie_ltssm (*get_ltssm)(struct dw_pcie *pcie);
 	int	(*start_link)(struct dw_pcie *pcie);
 	void	(*stop_link)(struct dw_pcie *pcie);
 };
@@ -393,6 +405,7 @@ struct dw_pcie {
 	struct reset_control_bulk_data	app_rsts[DW_PCIE_NUM_APP_RSTS];
 	struct reset_control_bulk_data	core_rsts[DW_PCIE_NUM_CORE_RSTS];
 	struct gpio_desc		*pe_rst;
+	bool			suspended;
 };
 
 #define to_dw_pcie_from_pp(port) container_of((port), struct dw_pcie, pp)
@@ -430,6 +443,9 @@ void dw_pcie_iatu_detect(struct dw_pcie *pci);
 int dw_pcie_edma_detect(struct dw_pcie *pci);
 void dw_pcie_edma_remove(struct dw_pcie *pci);
 
+int dw_pcie_suspend_noirq(struct dw_pcie *pci);
+int dw_pcie_resume_noirq(struct dw_pcie *pci);
+
 static inline void dw_pcie_writel_dbi(struct dw_pcie *pci, u32 reg, u32 val)
 {
 	dw_pcie_write_dbi(pci, reg, 0x4, val);
@@ -501,6 +517,18 @@ static inline void dw_pcie_stop_link(struct dw_pcie *pci)
 		pci->ops->stop_link(pci);
 }
 
+static inline enum dw_pcie_ltssm dw_pcie_get_ltssm(struct dw_pcie *pci)
+{
+	u32 val;
+
+	if (pci->ops && pci->ops->get_ltssm)
+		return pci->ops->get_ltssm(pci);
+
+	val = dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0);
+
+	return (enum dw_pcie_ltssm)FIELD_GET(PORT_LOGIC_LTSSM_STATE_MASK, val);
+}
+
 #ifdef CONFIG_PCIE_DW_HOST
 irqreturn_t dw_handle_msi_irq(struct dw_pcie_rp *pp);
 int dw_pcie_setup_rc(struct dw_pcie_rp *pp);
Signature · 1 line
2.34.1
Signature · 1 line
மணிவண்ணன் சதாசிவம்
Signature · 1 line
மணிவண்ணன் சதாசிவம்
#8 lore raw

Re: [PATCH v5 1/2] PCI: dwc: Implement general suspend/resume functionality for L2/L3 transitionse

In reply to Frank Li
Quoted 17 lines · Frank Li
Quoted 16 lines · Manivannan Sadhasivam

drivers/pci/controller/dwc/pcie-designware-host.c int dw_pcie_setup_rc(struct dw_pcie_rp *pp) · line 824

Quoted 12 lines · Frank Li

Introduced helper function dw_pcie_get_ltssm to retrieve SMLH_LTSS_STATE. Added API pme_turn_off and exit_from_l2 for managing L2/L3 state transitions.
...

drivers/pci/controller/dwc/pcie-designware-host.c +8 -0
+static void dw_pcie_set_dstate(struct dw_pcie *pci, pci_power_t dstate)
+{
+	u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_PM);
+	u16 val;
+
+	val = dw_pcie_readw_dbi(pci, offset + PCI_PM_CTRL);
+	val &= ~PCI_PM_CTRL_STATE_MASK;
+	val |= ((u16 __force)dstate) & PCI_PM_CTRL_STATE_MASK;

Why can't just,

Quoted 5 lines
Quoted 1 line

val |= dstate;

fixed a build warning.

This is the sparse warning from the lkp URL:

sparse warnings: (new ones prefixed by >>)

Quoted 3 lines
Quoted 3 lines

drivers/pci/controller/dwc/pcie-designware-host.c:824:13: sparse: sparse: invalid assignment: |=
drivers/pci/controller/dwc/pcie-designware-host.c:824:13: sparse: left side has type unsigned int
drivers/pci/controller/dwc/pcie-designware-host.c:824:13: sparse: right side has type restricted pci_power_t

We have a zillion of those warnings already (try "make C=2 drivers/pci/").

Personally I think it's better to omit the ugly cast and live with the warning for now. Someday somebody will figure a better way to use pci_power_t that will fix all these warnings at once. I'd rather wait for that fix than clutter all the uses with casts like this.

Bjorn