PCI: vmd: Enable ASPM if BIOS requests it
Subsystems (3)
RFC 1/1 PCI: vmd: Enable ASPM if BIOS requests it
VMD domains are not ACPI-managed devices and do not have the necessary ACPI hooks to enable ASPM. However if the BIOS has requested ASPM enablement, we should try to honor that request regardless. This patch adds the ASPM support to VMD child devices if requested by the FADT table.
---
Hi,
My knowledge on these kinds of power modes is limited, and we are having trouble bringing the Root Port child device out of L1 with this patch.
Can you help me understand the correct flow for bringing the Root Port device out of L1 with kernel flow, and what I might be missing here?
drivers/pci/controller/vmd.c | 9 ++++++++- drivers/pci/pcie/aspm.c | 19 ++----------------- include/linux/pci.h | 17 +++++++++++++++++ 3 files changed, 27 insertions(+), 18 deletions(-)
drivers/pci/controller/vmd.c +8 -1
@@ -14,6 +14,7 @@#include <linux/srcu.h> #include <linux/rculist.h> #include <linux/rcupdate.h> +#include <linux/acpi.h> #include <asm/irqdomain.h> #include <asm/device.h>
@@ -601,8 +602,14 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)* and will fail pcie_bus_configure_settings() early. It can instead be * run on each of the real root ports. */ -list_for_each_entry (child, &vmd->bus->children, node) +list_for_each_entry (child, &vmd->bus->children, node) { +#if IS_ENABLED (CONFIG_PCIEASPM ) +if (!(acpi_gbl_FADT.boot_flags &ACPI_FADT_NO_ASPM )) +pcie_config_aspm_link (child->self->link_state, +ASPM_STATE_ALL ); +#endif pcie_bus_configure_settings (child); + }pci_bus_add_devices (vmd->bus);
drivers/pci/pcie/aspm.c +2 -17
@@ -25,22 +25,6 @@#endif #define MODULE_PARAM_PREFIX "pcie_aspm." -/* Note: those are not register definitions */ -#define ASPM_STATE_L0S_UP (1)/* Upstream direction L0s state */ -#define ASPM_STATE_L0S_DW (2)/* Downstream direction L0s state */ -#define ASPM_STATE_L1 (4)/* L1 state */ -#define ASPM_STATE_L1_1 (8)/* ASPM L1.1 state */ -#define ASPM_STATE_L1_2 (0x10)/* ASPM L1.2 state */ -#define ASPM_STATE_L1_1_PCIPM (0x20)/* PCI PM L1.1 state */ -#define ASPM_STATE_L1_2_PCIPM (0x40)/* PCI PM L1.2 state */ -#define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM) -#define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM) -#define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\ - ASPM_STATE_L1_2_MASK) -#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) -#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \ - ASPM_STATE_L1SS) -struct aspm_latency {u32 l0s;/* L0s latency (nsec) */ u32 l1;/* L1 latency (nsec) */
@@ -748,7 +732,7 @@ static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) PCI_EXP_LNKCTL_ASPMC, val); } -static void pcie_config_aspm_link (struct pcie_link_state *link,u32 state) +void pcie_config_aspm_link (struct pcie_link_state *link,u32 state) {u32 upstream =0 , dwstream =0 ;struct pci_dev *child = link->downstream, *parent = link->pdev;
@@ -798,6 +782,7 @@ static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) link->aspm_enabled = state; } +EXPORT_SYMBOL_GPL (pcie_config_aspm_link);static void pcie_config_aspm_path (struct pcie_link_state *link) {
include/linux/pci.h +17 -0
@@ -377,6 +377,22 @@ struct pci_dev {unsigned int d3cold_delay;/* D3cold->D0 transition time in ms */ #ifdef CONFIG_PCIEASPM +/* Note: those are not register definitions */ +#define ASPM_STATE_L0S_UP (1)/* Upstream direction L0s state */ +#define ASPM_STATE_L0S_DW (2)/* Downstream direction L0s state */ +#define ASPM_STATE_L1 (4)/* L1 state */ +#define ASPM_STATE_L1_1 (8)/* ASPM L1.1 state */ +#define ASPM_STATE_L1_2 (0x10)/* ASPM L1.2 state */ +#define ASPM_STATE_L1_1_PCIPM (0x20)/* PCI PM L1.1 state */ +#define ASPM_STATE_L1_2_PCIPM (0x40)/* PCI PM L1.2 state */ +#define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM) +#define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM) +#define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\ + ASPM_STATE_L1_2_MASK) +#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) +#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \ + ASPM_STATE_L1SS) +struct pcie_link_state *link_state;/* ASPM link state */ unsigned int ltr_path:1; /* Latency Tolerance Reportingsupported from root to here */
@@ -1577,6 +1593,7 @@ extern bool pcie_ports_native;#define PCIE_LINK_STATE_L1_2_PCIPM BIT(6)#ifdef CONFIG_PCIEASPM +void pcie_config_aspm_link (struct pcie_link_state *link,u32 state);int pci_disable_link_state (struct pci_dev *pdev,int state);int pci_disable_link_state_locked (struct pci_dev *pdev,int state);void pcie_no_aspm (void );
Signature · 1 line
2.18.1
Quoted 5 lines · Jon Derrick
VMD domains are not ACPI-managed devices and do not have the necessary ACPI hooks to enable ASPM. However if the BIOS has requested ASPM enablement, we should try to honor that request regardless. This patch adds the ASPM support to VMD child devices if requested by the FADT table.
ASPM is enabled by software but the actual link state transitions between L0, L0s, L1, etc are done autonomously by hardware.
There should be no functional difference between ASPM being disabled vs being enabled. The only difference should be power consumption and some latency.
There are no ACPI hooks required to enable ASPM. The ACPI_FADT_NO_ASPM bit tells the OS that "it must not enable OSPM ASPM control" (ACPI v6.3, sec 5.2.9.3). AFAIK there is nothing in ACPI to request that the OS *should* enable ASPM.
Quoted 8 lines
Signed-off-by Jon Derrick <[email protected]>Signed-off-by You-Sheng Yang <[email protected]>---
Hi,
My knowledge on these kinds of power modes is limited, and we are having trouble bringing the Root Port child device out of L1 with this patch.
Presumably this patch helps something. Do you mean you're having trouble *without* this patch?
Quoted 2 lines
Can you help me understand the correct flow for bringing the Root Port device out of L1 with kernel flow, and what I might be missing here?
I don't understand the issue yet. L1 is a state of the link, not of an individual device. The devices on both ends of the link, e.g., a Root Port and an Endpoint or Switch Upstream Port below it, negotiate to determine the link state. This all happens in hardware without software involvement.
The only software influence is to enable hardware to select certain link states.
Quoted 18 lines
drivers/pci/controller/vmd.c | 9 ++++++++- drivers/pci/pcie/aspm.c | 19 ++----------------- include/linux/pci.h | 17 +++++++++++++++++ 3 files changed, 27 insertions(+), 18 deletions(-)drivers/pci/controller/vmd.c +1
-0@@ -14,6 +14,7 @@#include <linux/srcu.h> #include <linux/rculist.h> #include <linux/rcupdate.h> +#include <linux/acpi.h> #include <asm/irqdomain.h> #include <asm/device.h>
drivers/pci/controller/vmd.c static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) · line 610
Quoted 10 lines
drivers/pci/controller/vmd.c +6
-1@@ -601,8 +602,14 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)* and will fail pcie_bus_configure_settings() early. It can instead be * run on each of the real root ports. */ -list_for_each_entry (child, &vmd->bus->children, node) +list_for_each_entry (child, &vmd->bus->children, node) { +#if IS_ENABLED (CONFIG_PCIEASPM ) +if (!(acpi_gbl_FADT.boot_flags &ACPI_FADT_NO_ASPM )) +pcie_config_aspm_link (child->self->link_state, +ASPM_STATE_ALL ); +#endif
pcie_aspm_init_link_state() is called from pci_scan_slot() and should be doing ASPM configuration without help from VMD. If that's not happening, I would instrument pcie_aspm_init_link_state() to make sure it's being called and to figure out if there's something there that prevents ASPM config for children of the VMD device.
Quoted 87 lines
drivers/pci/controller/vmd.c +1
-0pcie_bus_configure_settings (child); + }pci_bus_add_devices (vmd->bus);drivers/pci/pcie/aspm.c +2
-17@@ -25,22 +25,6 @@#endif #define MODULE_PARAM_PREFIX "pcie_aspm." -/* Note: those are not register definitions */ -#define ASPM_STATE_L0S_UP (1)/* Upstream direction L0s state */ -#define ASPM_STATE_L0S_DW (2)/* Downstream direction L0s state */ -#define ASPM_STATE_L1 (4)/* L1 state */ -#define ASPM_STATE_L1_1 (8)/* ASPM L1.1 state */ -#define ASPM_STATE_L1_2 (0x10)/* ASPM L1.2 state */ -#define ASPM_STATE_L1_1_PCIPM (0x20)/* PCI PM L1.1 state */ -#define ASPM_STATE_L1_2_PCIPM (0x40)/* PCI PM L1.2 state */ -#define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM) -#define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM) -#define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\ - ASPM_STATE_L1_2_MASK) -#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) -#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \ - ASPM_STATE_L1SS) -struct aspm_latency {u32 l0s;/* L0s latency (nsec) */ u32 l1;/* L1 latency (nsec) */ @@ -748,7 +732,7 @@ static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) PCI_EXP_LNKCTL_ASPMC, val); } -static void pcie_config_aspm_link (struct pcie_link_state *link,u32 state) +void pcie_config_aspm_link (struct pcie_link_state *link,u32 state) {u32 upstream =0 , dwstream =0 ;struct pci_dev *child = link->downstream, *parent = link->pdev;@@ -798,6 +782,7 @@ static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) link->aspm_enabled = state; } +EXPORT_SYMBOL_GPL (pcie_config_aspm_link);static void pcie_config_aspm_path (struct pcie_link_state *link) {include/linux/pci.h +17
-0@@ -377,6 +377,22 @@ struct pci_dev {unsigned int d3cold_delay;/* D3cold->D0 transition time in ms */ #ifdef CONFIG_PCIEASPM +/* Note: those are not register definitions */ +#define ASPM_STATE_L0S_UP (1)/* Upstream direction L0s state */ +#define ASPM_STATE_L0S_DW (2)/* Downstream direction L0s state */ +#define ASPM_STATE_L1 (4)/* L1 state */ +#define ASPM_STATE_L1_1 (8)/* ASPM L1.1 state */ +#define ASPM_STATE_L1_2 (0x10)/* ASPM L1.2 state */ +#define ASPM_STATE_L1_1_PCIPM (0x20)/* PCI PM L1.1 state */ +#define ASPM_STATE_L1_2_PCIPM (0x40)/* PCI PM L1.2 state */ +#define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM) +#define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM) +#define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\ + ASPM_STATE_L1_2_MASK) +#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) +#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \ + ASPM_STATE_L1SS) +struct pcie_link_state *link_state;/* ASPM link state */ unsigned int ltr_path:1; /* Latency Tolerance Reportingsupported from root to here */ @@ -1577,6 +1593,7 @@ extern bool pcie_ports_native;#define PCIE_LINK_STATE_L1_2_PCIPM BIT(6)#ifdef CONFIG_PCIEASPM +void pcie_config_aspm_link (struct pcie_link_state *link,u32 state);int pci_disable_link_state (struct pci_dev *pdev,int state);int pci_disable_link_state_locked (struct pci_dev *pdev,int state);void pcie_no_aspm (void );Signature · 1 line
2.18.1
Sorry for the delay. New patch incoming
Quoted 7 lines · Bjorn Helgaas
Quoted 5 lines · Jon Derrick
VMD domains are not ACPI-managed devices and do not have the necessary ACPI hooks to enable ASPM. However if the BIOS has requested ASPM enablement, we should try to honor that request regardless. This patch adds the ASPM support to VMD child devices if requested by the FADT table.
Quoted 6 lines
ASPM is enabled by software but the actual link state transitions between L0, L0s, L1, etc are done autonomously by hardware.
There should be no functional difference between ASPM being disabled vs being enabled. The only difference should be power consumption and some latency.
It seems there was some confusion on my end about the role of BIOS in setting ASPM defaults.
Quoted 6 lines
There are no ACPI hooks required to enable ASPM. The
ACPI_FADT_NO_ASPMbit tells the OS that "it must not enable OSPM ASPM control" (ACPI v6.3, sec 5.2.9.3). AFAIK there is nothing in ACPI to request that the OS *should* enable ASPM.
I see how it's used and not needed. Thanks!
Quoted 15 lines
Quoted 8 lines
Signed-off-by Jon Derrick <[email protected]>Signed-off-by You-Sheng Yang <[email protected]>---
Hi,
My knowledge on these kinds of power modes is limited, and we are having trouble bringing the Root Port child device out of L1 with this patch.
Presumably this patch helps something. Do you mean you're having trouble *without* this patch?
Quoted 2 lines
Can you help me understand the correct flow for bringing the Root Port device out of L1 with kernel flow, and what I might be missing here?
Quoted 8 lines
I don't understand the issue yet. L1 is a state of the link, not of an individual device. The devices on both ends of the link, e.g., a Root Port and an Endpoint or Switch Upstream Port below it, negotiate to determine the link state. This all happens in hardware without software involvement.
The only software influence is to enable hardware to select certain link states.
The confusion seemed to be about enabling the link states, which seemed to have been cleared up as a BIOS responsibility for default ASPM policy.
Quoted 123 lines
Quoted 19 lines
drivers/pci/controller/vmd.c | 9 ++++++++- drivers/pci/pcie/aspm.c | 19 ++----------------- include/linux/pci.h | 17 +++++++++++++++++ 3 files changed, 27 insertions(+), 18 deletions(-)drivers/pci/controller/vmd.c +1
-0@@ -14,6 +14,7 @@#include <linux/srcu.h> #include <linux/rculist.h> #include <linux/rcupdate.h> +#include <linux/acpi.h> #include <asm/irqdomain.h> #include <asm/device.h> drivers/pci/controller/vmd.c static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) · line 610
Quoted 10 lines
drivers/pci/controller/vmd.c +6
-1@@ -601,8 +602,14 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)* and will fail pcie_bus_configure_settings() early. It can instead be * run on each of the real root ports. */ -list_for_each_entry (child, &vmd->bus->children, node) +list_for_each_entry (child, &vmd->bus->children, node) { +#if IS_ENABLED (CONFIG_PCIEASPM ) +if (!(acpi_gbl_FADT.boot_flags &ACPI_FADT_NO_ASPM )) +pcie_config_aspm_link (child->self->link_state, +ASPM_STATE_ALL ); +#endif
pcie_aspm_init_link_state()is called frompci_scan_slot()and should be doing ASPM configuration without help from VMD. If that's not happening, I would instrumentpcie_aspm_init_link_state()to make sure it's being called and to figure out if there's something there that prevents ASPM config for children of the VMD device.Quoted 87 lines
drivers/pci/controller/vmd.c +1
-0pcie_bus_configure_settings (child); + }pci_bus_add_devices (vmd->bus);drivers/pci/pcie/aspm.c +2
-17@@ -25,22 +25,6 @@#endif #define MODULE_PARAM_PREFIX "pcie_aspm." -/* Note: those are not register definitions */ -#define ASPM_STATE_L0S_UP (1)/* Upstream direction L0s state */ -#define ASPM_STATE_L0S_DW (2)/* Downstream direction L0s state */ -#define ASPM_STATE_L1 (4)/* L1 state */ -#define ASPM_STATE_L1_1 (8)/* ASPM L1.1 state */ -#define ASPM_STATE_L1_2 (0x10)/* ASPM L1.2 state */ -#define ASPM_STATE_L1_1_PCIPM (0x20)/* PCI PM L1.1 state */ -#define ASPM_STATE_L1_2_PCIPM (0x40)/* PCI PM L1.2 state */ -#define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM) -#define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM) -#define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\ - ASPM_STATE_L1_2_MASK) -#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) -#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \ - ASPM_STATE_L1SS) -struct aspm_latency {u32 l0s;/* L0s latency (nsec) */ u32 l1;/* L1 latency (nsec) */ @@ -748,7 +732,7 @@ static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) PCI_EXP_LNKCTL_ASPMC, val); } -static void pcie_config_aspm_link (struct pcie_link_state *link,u32 state) +void pcie_config_aspm_link (struct pcie_link_state *link,u32 state) {u32 upstream =0 , dwstream =0 ;struct pci_dev *child = link->downstream, *parent = link->pdev;@@ -798,6 +782,7 @@ static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) link->aspm_enabled = state; } +EXPORT_SYMBOL_GPL (pcie_config_aspm_link);static void pcie_config_aspm_path (struct pcie_link_state *link) {include/linux/pci.h +17
-0@@ -377,6 +377,22 @@ struct pci_dev {unsigned int d3cold_delay;/* D3cold->D0 transition time in ms */ #ifdef CONFIG_PCIEASPM +/* Note: those are not register definitions */ +#define ASPM_STATE_L0S_UP (1)/* Upstream direction L0s state */ +#define ASPM_STATE_L0S_DW (2)/* Downstream direction L0s state */ +#define ASPM_STATE_L1 (4)/* L1 state */ +#define ASPM_STATE_L1_1 (8)/* ASPM L1.1 state */ +#define ASPM_STATE_L1_2 (0x10)/* ASPM L1.2 state */ +#define ASPM_STATE_L1_1_PCIPM (0x20)/* PCI PM L1.1 state */ +#define ASPM_STATE_L1_2_PCIPM (0x40)/* PCI PM L1.2 state */ +#define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM) +#define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM) +#define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\ + ASPM_STATE_L1_2_MASK) +#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) +#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \ + ASPM_STATE_L1SS) +struct pcie_link_state *link_state;/* ASPM link state */ unsigned int ltr_path:1; /* Latency Tolerance Reportingsupported from root to here */ @@ -1577,6 +1593,7 @@ extern bool pcie_ports_native;#define PCIE_LINK_STATE_L1_2_PCIPM BIT(6)#ifdef CONFIG_PCIEASPM +void pcie_config_aspm_link (struct pcie_link_state *link,u32 state);int pci_disable_link_state (struct pci_dev *pdev,int state);int pci_disable_link_state_locked (struct pci_dev *pdev,int state);void pcie_no_aspm (void );Signature · 1 line
2.18.1