Skip to contentSkip to conversation

Runtime PM for Thunderbolt on Macs

PATCH Superseded Lukas Wunner 3 participants 17 messages Jan 8, 2017 – Jan 15, 2017

Subsystems (6)

v4 of 6 Superseded Latest is v6 →

lore
Share & tools
lore
#0 lore raw

v4 cover Runtime PM for Thunderbolt on Macs

Power down Thunderbolt controllers on Macs when nothing is plugged in to save around 2W per controller.

For details see the cover letter of v3:
https://lkml.org/lkml/2016/12/17/56

Patches [1/7] to [3/7] need an ack from Bjorn (and possibly Rafael or Mika). Patches [4/7] to [6/7] need an ack from Rafael.

Changes since v3:

- Additional patch [6/8] by Chen Yu to define a DPM_DIRECT_COMPLETE macro. I had expected this to land in v4.10 but it didn't.

- Rework patch [7/8] ("thunderbolt: Power down controller when idle") according to Andy Shevchenko's feedback: Drop unnecessary #ifdef pr_fmt, add explanatory comments, rename goto labels.

- Fix error path if the upstream bridge cannot be found.

The patches are also browseable on GitHub:
https://github.com/l1k/linux/commits/thunderbolt_runpm_v4

Thanks,

Lukas

Chen Yu (1):
PM / sleep: Define constant for direct_complete

Lukas Wunner (7):
PCI: Recognize Thunderbolt devices
PCI: Allow runtime PM on Thunderbolt ports
PCI: Don't block runtime PM for Thunderbolt host hotplug ports Revert "PM / Runtime: Remove the exported function
pm_children_suspended()"
PM: Make requirements of dev_pm_domain_set() more precise
thunderbolt: Power down controller when idle
thunderbolt: Runtime suspend NHI when idle

 drivers/base/power/common.c  |  15 +-
 drivers/base/power/runtime.c |   3 +-
 drivers/pci/pci.c            |  20 ++-
 drivers/pci/pci.h            |   2 +
 drivers/pci/probe.c          |  34 +++++
 drivers/thunderbolt/Kconfig  |   3 +-
 drivers/thunderbolt/Makefile |   4 +-
 drivers/thunderbolt/nhi.c    |   5 +
 drivers/thunderbolt/power.c  | 355 +++++++++++++++++++++++++++++++++++++++++++
 drivers/thunderbolt/power.h  |  37 +++++
 drivers/thunderbolt/switch.c |   9 ++
 drivers/thunderbolt/tb.c     |  13 ++
 drivers/thunderbolt/tb.h     |   2 +
 include/linux/pci.h          |   1 +
 include/linux/pm.h           |   7 +
 include/linux/pm_runtime.h   |   7 +
 16 files changed, 506 insertions(+), 11 deletions(-)
 create mode 100644 drivers/thunderbolt/power.c
 create mode 100644 drivers/thunderbolt/power.h
Signature · 1 line
2.11.0
#1 lore raw

PATCH v4 1/8 PCI: Recognize Thunderbolt devices

We're about to allow runtime PM on Thunderbolt ports in pci_bridge_d3_possible() and unblock runtime PM for Thunderbolt host hotplug ports in pci_dev_check_d3cold(). In both cases we need to uniquely identify if a PCI device belongs to a Thunderbolt controller.

We also have the need to detect presence of a Thunderbolt controller in drivers/platform/x86/apple-gmux.c because dual GPU MacBook Pros cannot switch external DP/HDMI ports between GPUs if they have Thunderbolt.

Furthermore, in multiple places in the DRM subsystem we need to detect whether a GPU is on-board or attached with Thunderbolt. As an example, Thunderbolt-attached GPUs shall not be registered with vga_switcheroo.

Intel uses a Vendor-Specific Extended Capability (VSEC) with ID 0x1234 on devices belonging to a Thunderbolt controller which allows us to recognize them.

Detect presence of this VSEC on device probe and cache it in a newly added is_thunderbolt bit in struct pci_dev which can then be queried by pci_bridge_d3_possible(), pci_dev_check_d3cold(), apple-gmux and others.

Cc Andreas Noever <[email protected]>
Cc Tomas Winkler <[email protected]>
Cc Amir Levy <[email protected]>
---
 drivers/pci/pci.h   |  2 ++
 drivers/pci/probe.c | 34 ++++++++++++++++++++++++++++++++++
 include/linux/pci.h |  1 +
 3 files changed, 37 insertions(+)
drivers/pci/pci.h +2 -0
index cb17db2..45c2b81 100644
@@ -3,6 +3,8 @@
 
 #define PCI_FIND_CAP_TTL	48
 
+#define PCI_VSEC_ID_INTEL_TBT	0x1234	/* Thunderbolt */
+
 extern const unsigned char pcie_link_speed[];
 
 bool pcie_cap_has_lnkctl(const struct pci_dev *dev);
drivers/pci/probe.c +34 -0
index e164b5c..891a8fa 100644
@@ -1206,6 +1206,37 @@ void set_pcie_hotplug_bridge(struct pci_dev *pdev)
 		pdev->is_hotplug_bridge = 1;
 }
 
+static void set_pcie_vendor_specific(struct pci_dev *dev)
+{
+	int vsec = 0;
+	u32 header;
+
+	while ((vsec = pci_find_next_ext_capability(dev, vsec,
+						    PCI_EXT_CAP_ID_VNDR))) {
+		pci_read_config_dword(dev, vsec + PCI_VNDR_HEADER, &header);
+
+		/* Is the device part of a Thunderbolt controller? */
+		if (dev->vendor == PCI_VENDOR_ID_INTEL &&
+		    PCI_VNDR_HEADER_ID(header) == PCI_VSEC_ID_INTEL_TBT)
+			dev->is_thunderbolt = 1;
+	}
+
+	/*
+	 * Is the device attached with Thunderbolt?  Walk upwards and check for
+	 * each encountered bridge if it's part of a Thunderbolt controller.
+	 * Reaching the host bridge means dev is soldered to the mainboard.
+	 */
+	if (!dev->is_thunderbolt) {
+		struct pci_dev *parent = dev;
+
+		while ((parent = pci_upstream_bridge(parent)))
+			if (parent->is_thunderbolt) {
+				dev->is_thunderbolt = 1;
+				break;
+			}
+	}
+}
+
 /**
  * pci_ext_cfg_is_aliased - is ext config space just an alias of std config?
  * @dev: PCI device
@@ -1358,6 +1389,9 @@ int pci_setup_device(struct pci_dev *dev)
 	/* need to have dev->class ready */
 	dev->cfg_size = pci_cfg_space_size(dev);
 
+	/* need to have dev->cfg_size ready */
+	set_pcie_vendor_specific(dev);
+
 	/* "Unknown power state" */
 	dev->current_state = PCI_UNKNOWN;
 
include/linux/pci.h +1 -0
index e2d1a12..3c775e8 100644
@@ -358,6 +358,7 @@ struct pci_dev {
 	unsigned int	is_virtfn:1;
 	unsigned int	reset_fn:1;
 	unsigned int    is_hotplug_bridge:1;
+	unsigned int	is_thunderbolt:1; /* part of Thunderbolt daisy chain */
 	unsigned int    __aer_firmware_first_valid:1;
 	unsigned int	__aer_firmware_first:1;
 	unsigned int	broken_intx_masking:1;
Signature · 1 line
2.11.0
#2 lore raw

RE: [PATCH v4 1/8] PCI: Recognize Thunderbolt devices

Quoted 4 lines

=20
We're about to allow runtime PM on Thunderbolt ports in pci_bridge_d3_possible() and unblock runtime PM for Thunderbolt host hotplug ports in pci_dev_check_d3cold(). In both cases we need to unique=

ly

Quoted 4 lines

identify if a PCI device belongs to a Thunderbolt controller. =20
We also have the need to detect presence of a Thunderbolt controller in drivers/platform/x86/apple-gmux.c because dual GPU MacBook Pros cannot

Quoted 8 lines

switch external DP/HDMI ports between GPUs if they have Thunderbolt. =20
Furthermore, in multiple places in the DRM subsystem we need to detect whether a GPU is on-board or attached with Thunderbolt. As an example, Thunderbolt-attached GPUs shall not be registered with vga_switcheroo. =20
Intel uses a Vendor-Specific Extended Capability (VSEC) with ID 0x1234 on devices belonging to a Thunderbolt controller which allows us to recogniz=

e

Quoted 3 lines

them.
=20
Detect presence of this VSEC on device probe and cache it in a newly adde=

d

Quoted 18 lines

is_thunderbolt bit in struct pci_dev which can then be queried by pci_bridge_d3_possible(), pci_dev_check_d3cold(), apple-gmux and others. =20

Cc Andreas Noever <[email protected]>
Cc Tomas Winkler <[email protected]>
Cc Amir Levy <[email protected]>
---
 drivers/pci/pci.h   |  2 ++
 drivers/pci/probe.c | 34 ++++++++++++++++++++++++++++++++++
 include/linux/pci.h |  1 +
 3 files changed, 37 insertions(+)

=20

drivers/pci/pci.h index cb17db2..45c2b81 renamed from drivers/pci/pci.h +0 -0

100644

drivers/pci/pci.h +0 -0
@@ -3,6 +3,8 @@
Quoted 4 lines

=20
#define PCI_FIND_CAP_TTL 48
=20
+#define PCI_VSEC_ID_INTEL_TBT 0x1234 /* Thunderbolt */

Shouldn't bet this defined in pci_ids.h ?=20

Quoted 10 lines

extern const unsigned char pcie_link_speed[];
=20
bool pcie_cap_has_lnkctl(const struct pci_dev *dev); diff --git
a/drivers/pci/probe.c b/drivers/pci/probe.c index e164b5c..891a8fa 100644

drivers/pci/probe.c +0 -0
@@ -1206,6 +1206,37 @@ void set_pcie_hotplug_bridge(struct pci_dev
 *pdev)
 		pdev->is_hotplug_bridge =3D 1;
 }
Quoted 2 lines

=20
+static void set_pcie_vendor_specific(struct pci_dev *dev) {

Why don't u implement this in quirk.c ?

Quoted 23 lines
drivers/pci/probe.c +6 -0
+	int vsec =3D 0;
+	u32 header;
+
+	while ((vsec =3D pci_find_next_ext_capability(dev, vsec,
+						    PCI_EXT_CAP_ID_VNDR))) {
+		pci_read_config_dword(dev, vsec + PCI_VNDR_HEADER,

&header);

drivers/pci/probe.c +9 -0
+
+		/* Is the device part of a Thunderbolt controller? */
+		if (dev->vendor =3D=3D PCI_VENDOR_ID_INTEL &&
+		    PCI_VNDR_HEADER_ID(header) =3D=3D PCI_VSEC_ID_INTEL_TBT)
+			dev->is_thunderbolt =3D 1;
+	}
+
+	/*
+	 * Is the device attached with Thunderbolt?  Walk upwards and check

for

drivers/pci/probe.c +6 -0
+	 * each encountered bridge if it's part of a Thunderbolt controller.
+	 * Reaching the host bridge means dev is soldered to the mainboard.
+	 */
+	if (!dev->is_thunderbolt) {
+		struct pci_dev *parent =3D dev;
+

drivers/pci/probe.c void set_pcie_hotplug_bridge(struct pci_dev *pdev)

Quoted 10 lines
drivers/pci/probe.c +8 -0
+		while ((parent =3D pci_upstream_bridge(parent)))
+			if (parent->is_thunderbolt) {
+				dev->is_thunderbolt =3D 1;
+				break;
+			}
+	}
+}
+
 /**
  * pci_ext_cfg_is_aliased - is ext config space just an alias of std con=

fig?

include/linux/pci.h index e2d1a12..3c7=

Quoted 12 lines

* @dev: PCI device

drivers/pci/probe.c +0 -0
@@ -1358,6 +1389,9 @@ int pci_setup_device(struct pci_dev *dev)
 	/* need to have dev->class ready */
 	dev->cfg_size =3D pci_cfg_space_size(dev);

=20

drivers/pci/probe.c +3 -0
+	/* need to have dev->cfg_size ready */
+	set_pcie_vendor_specific(dev);
+
 	/* "Unknown power state" */
 	dev->current_state =3D PCI_UNKNOWN;

=20

include/linux/pci.h index e2d1a12..3c7= renamed from include/linux/pci.h +0 -0

75e8

Quoted 14 lines

100644

include/linux/pci.h +1 -0
@@ -358,6 +358,7 @@ struct pci_dev {
 	unsigned int	is_virtfn:1;
 	unsigned int	reset_fn:1;
 	unsigned int    is_hotplug_bridge:1;
+	unsigned int	is_thunderbolt:1; /* part of Thunderbolt daisy chain
 */
 	unsigned int    __aer_firmware_first_valid:1;
 	unsigned int	__aer_firmware_first:1;

unsigned int broken_intx_masking:1;
--
2.11.0

#3 lore raw

Re: [PATCH v4 1/8] PCI: Recognize Thunderbolt devices

Quoted 37 lines · Tomas
Quoted 37 lines

We're about to allow runtime PM on Thunderbolt ports in pci_bridge_d3_possible() and unblock runtime PM for Thunderbolt host hotplug ports in pci_dev_check_d3cold(). In both cases we need to uniquely identify if a PCI device belongs to a Thunderbolt controller.

We also have the need to detect presence of a Thunderbolt controller in drivers/platform/x86/apple-gmux.c because dual GPU MacBook Pros cannot switch external DP/HDMI ports between GPUs if they have Thunderbolt.

Furthermore, in multiple places in the DRM subsystem we need to detect whether a GPU is on-board or attached with Thunderbolt. As an example, Thunderbolt-attached GPUs shall not be registered with vga_switcheroo.

Intel uses a Vendor-Specific Extended Capability (VSEC) with ID 0x1234 on devices belonging to a Thunderbolt controller which allows us to recognize them.

Detect presence of this VSEC on device probe and cache it in a newly added is_thunderbolt bit in struct pci_dev which can then be queried by pci_bridge_d3_possible(), pci_dev_check_d3cold(), apple-gmux and others.

Cc Andreas Noever <[email protected]>
Cc Tomas Winkler <[email protected]>
Cc Amir Levy <[email protected]>
---
 drivers/pci/pci.h   |  2 ++
 drivers/pci/probe.c | 34 ++++++++++++++++++++++++++++++++++
 include/linux/pci.h |  1 +
 3 files changed, 37 insertions(+)
drivers/pci/pci.h index cb17db2..45c2b81 renamed from drivers/pci/pci.h +0 -0

100644

drivers/pci/pci.h +0 -0
@@ -3,6 +3,8 @@
 
Quoted 5 lines

drivers/pci/pci.h line 6

Quoted 3 lines
drivers/pci/pci.h +1 -0
 #define PCI_FIND_CAP_TTL	48
 
+#define PCI_VSEC_ID_INTEL_TBT	0x1234	/* Thunderbolt */

Shouldn't bet this defined in pci_ids.h ?

That file is solely intended for IDs used in multiple places, as explained in the comment at its top. This particular ID is only used in the PCI core, hence it's in the PCI core's private drivers/pci/pci.h.

However the line is formatted such that it can just be moved to the global include/linux/pci_ids.h should it be needed someplace else in the future.

Quoted 11 lines
Quoted 11 lines
drivers/pci/pci.h +0 -0
 extern const unsigned char pcie_link_speed[];
 
 bool pcie_cap_has_lnkctl(const struct pci_dev *dev); diff --git

a/drivers/pci/probe.c b/drivers/pci/probe.c index e164b5c..891a8fa 100644

drivers/pci/probe.c +0 -0
@@ -1206,6 +1206,37 @@ void set_pcie_hotplug_bridge(struct pci_dev
 *pdev)
 		pdev->is_hotplug_bridge = 1;
 }
 
Quoted 4 lines

drivers/pci/probe.c void set_pcie_hotplug_bridge(struct pci_dev · line 1210

Quoted 1 line
drivers/pci/probe.c +1 -0
+static void set_pcie_vendor_specific(struct pci_dev *dev) {

Why don't u implement this in quirk.c ?

The is_thunderbolt bit signifies whether a given PCI device is part of a Thunderbolt daisy chain. (As explained in the comment in include/linux/pci.h, see below.) So it is set on all the PCI devices that comprise a Thunderbolt controller, but also on all endpoint devices that are attached via Thunderbolt.

Consequently this function needs to be executed for all PCI devices, not just for a subset that could be identified by a vendor, device or class ID.

In the DRM drivers nouveau, radeon and amdgpu I need to prevent calls to vga_switcheroo_register_client() and vga_switcheroo_init_domain_pm_ops() if the device in question is attached via Thunderbolt. That's because external GPUs can't drive the panel of a laptop or be powered down like an on-board Optimus/PowerXpress GPU. We've got a bug there right now that manifests itself once someone attaches an eGPU to a dual GPU laptop. With this patch I'll be able to just skip registration with vga_switcheroo if is_thunderbolt is set on a GPU's pci_dev, thereby fixing the bug.

BTW do you have information on the contents/meaning of the VSEC that you would be able/allowed to share? What I've seen so far is that its length is 0x1c bytes on older controllers (e.g. Light Ridge, Port Ridge) and its contents are always the same, regardless if the controller is used in host mode or endpoint mode:

500: 0b 00 01 00 34 12 c1 01|50 09 00 00 39 00 00 00 510: 00 00 00 00 00 00 00 00 00 00 00 00

However on Alpine Ridge the size of the VSEC has increased significantly to 0xd8 bytes, even though the version stayed at 1 as before:

500: 0b 00 01 60 34 12 81 0d 50 09 b0 0c b9 06 18 08 510: 00 38 00 01 00 00 00 00 00 00 00 00 32 02 10 00 520: ef d3 00 40 00 00 00 00 f0 f0 30 c1 00 02 30 00 530: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 540: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 550: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 560: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 570: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 580: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 590: 00 00 00 08 00 00 00 00 00 00 00 00 00 00 00 00 5a0: 00 00 00 00 00 00 00 00 00 00 00 00 38 24 01 00 5b0: 08 40 00 1c 00 00 01 18 04 03 00 f0 06 00 00 00 5c0: 00 00 08 40 06 00 00 01 90 00 08 1b 00 08 80 08 5d0: 80 7f 08 00 00 00 00 00

Thanks,

Lukas

Quoted 59 lines
Quoted 59 lines
drivers/pci/probe.c +15 -0
+	int vsec = 0;
+	u32 header;
+
+	while ((vsec = pci_find_next_ext_capability(dev, vsec,
+						    PCI_EXT_CAP_ID_VNDR))) {
+		pci_read_config_dword(dev, vsec + PCI_VNDR_HEADER,
 &header);
+
+		/* Is the device part of a Thunderbolt controller? */
+		if (dev->vendor == PCI_VENDOR_ID_INTEL &&
+		    PCI_VNDR_HEADER_ID(header) == PCI_VSEC_ID_INTEL_TBT)
+			dev->is_thunderbolt = 1;
+	}
+
+	/*
+	 * Is the device attached with Thunderbolt?  Walk upwards and check

for

drivers/pci/probe.c +17 -0
+	 * each encountered bridge if it's part of a Thunderbolt controller.
+	 * Reaching the host bridge means dev is soldered to the mainboard.
+	 */
+	if (!dev->is_thunderbolt) {
+		struct pci_dev *parent = dev;
+
+		while ((parent = pci_upstream_bridge(parent)))
+			if (parent->is_thunderbolt) {
+				dev->is_thunderbolt = 1;
+				break;
+			}
+	}
+}
+
 /**
  * pci_ext_cfg_is_aliased - is ext config space just an alias of std config?
  * @dev: PCI device
@@ -1358,6 +1389,9 @@ int pci_setup_device(struct pci_dev *dev)
 	/* need to have dev->class ready */
 	dev->cfg_size = pci_cfg_space_size(dev);
 
+	/* need to have dev->cfg_size ready */
+	set_pcie_vendor_specific(dev);
+
 	/* "Unknown power state" */
 	dev->current_state = PCI_UNKNOWN;
 
include/linux/pci.h index e2d1a12..3c775e8 renamed from include/linux/pci.h +0 -0

100644

include/linux/pci.h +1 -1
@@ -358,6 +358,7 @@ struct pci_dev {
 	unsigned int	is_virtfn:1;
 	unsigned int	reset_fn:1;
 	unsigned int    is_hotplug_bridge:1;
+	unsigned int	is_thunderbolt:1; /* part of Thunderbolt daisy chain */
 	unsigned int    __aer_firmware_first_valid:1;
 	unsigned int	__aer_firmware_first:1;
 	unsigned int	broken_intx_masking:1;
--

2.11.0

#4 lore raw

RE: [PATCH v4 1/8] PCI: Recognize Thunderbolt devices

Quoted 6 lines

=20

Quoted 4 lines · Tomas
Quoted 4 lines

We're about to allow runtime PM on Thunderbolt ports in pci_bridge_d3_possible() and unblock runtime PM for Thunderbolt host hotplug ports in pci_dev_check_d3cold(). In both cases we need to uniquely identify if a PCI device belongs to a Thunderbolt controller=

.

Quoted 37 lines
Quoted 4 lines
Quoted 4 lines

We also have the need to detect presence of a Thunderbolt controller in drivers/platform/x86/apple-gmux.c because dual GPU MacBook Pros cannot switch external DP/HDMI ports between GPUs if they have

Thunderbolt.

Quoted 4 lines
Quoted 4 lines

Furthermore, in multiple places in the DRM subsystem we need to detect whether a GPU is on-board or attached with Thunderbolt. As an example, Thunderbolt-attached GPUs shall not be registered with

vga_switcheroo.

Quoted 8 lines
Quoted 8 lines

Intel uses a Vendor-Specific Extended Capability (VSEC) with ID 0x1234 on devices belonging to a Thunderbolt controller which allows us to recognize them.

Detect presence of this VSEC on device probe and cache it in a newly added is_thunderbolt bit in struct pci_dev which can then be queried by pci_bridge_d3_possible(), pci_dev_check_d3cold(), apple-gmux and

others.

Quoted 18 lines
Quoted 18 lines
Cc Andreas Noever <[email protected]>
Cc Tomas Winkler <[email protected]>
Cc Amir Levy <[email protected]>
---
 drivers/pci/pci.h   |  2 ++
 drivers/pci/probe.c | 34 ++++++++++++++++++++++++++++++++++
 include/linux/pci.h |  1 +
 3 files changed, 37 insertions(+)
drivers/pci/pci.h index renamed from drivers/pci/pci.h +0 -0

cb17db2..45c2b81
100644

drivers/pci/pci.h +0 -0
@@ -3,6 +3,8 @@
 
Quoted 7 lines
Quoted 5 lines

drivers/pci/pci.h line 6

Quoted 3 lines
drivers/pci/pci.h +1 -0
 #define PCI_FIND_CAP_TTL	48
 
+#define PCI_VSEC_ID_INTEL_TBT	0x1234	/* Thunderbolt */

Shouldn't bet this defined in pci_ids.h ?

=20
That file is solely intended for IDs used in multiple places, as explaine=

d in the

Quoted 1 line

comment at its top. This particular ID is only used in the PCI core, hen=

ce it's in

Quoted 3 lines

the PCI core's private drivers/pci/pci.h.
=20
However the line is formatted such that it can just be moved to the globa=

l

Quoted 15 lines

include/linux/pci_ids.h should it be needed someplace else in the future. =20
=20

Quoted 12 lines
Quoted 12 lines
drivers/pci/pci.h +0 -0
 extern const unsigned char pcie_link_speed[];
 
 bool pcie_cap_has_lnkctl(const struct pci_dev *dev); diff --git

a/drivers/pci/probe.c b/drivers/pci/probe.c index e164b5c..891a8fa 100644

drivers/pci/probe.c +0 -0
@@ -1206,6 +1206,37 @@ void set_pcie_hotplug_bridge(struct pci_dev
 *pdev)
 		pdev->is_hotplug_bridge =3D 1;
 }
 
Quoted 7 lines
Quoted 4 lines

drivers/pci/probe.c void set_pcie_hotplug_bridge(struct pci_dev · line 1210

Quoted 1 line
drivers/pci/probe.c +1 -0
+static void set_pcie_vendor_specific(struct pci_dev *dev) {

Why don't u implement this in quirk.c ?

=20
The is_thunderbolt bit signifies whether a given PCI device is part of a Thunderbolt daisy chain. (As explained in the comment in include/linux/p=

ci.h,

Quoted 1 line

see below.) So it is set on all the PCI devices that comprise a Thunderb=

olt

Quoted 1 line

controller, but also on all endpoint devices that are attached via Thunde=

rbolt.

Quoted 2 lines

=20
Consequently this function needs to be executed for all PCI devices, not =

just

Quoted 1 line

for a subset that could be identified by a vendor, device or class ID.

Ok, god it.=20

Quoted 4 lines

=20
In the DRM drivers nouveau, radeon and amdgpu I need to prevent calls to vga_switcheroo_register_client() and vga_switcheroo_init_domain_pm_ops() if the device in question is attached via Thunderbolt. That's because ex=

ternal

Quoted 1 line

GPUs can't drive the panel of a laptop or be powered down like an on-boar=

d

Quoted 3 lines

Optimus/PowerXpress GPU. We've got a bug there right now that manifests itself once someone attaches an eGPU to a dual GPU laptop. With this patch I'll be able to just skip registration with vga_switchero=

o if

Quoted 5 lines

is_thunderbolt is set on a GPU's pci_dev, thereby fixing the bug. =20
=20
BTW do you have information on the contents/meaning of the VSEC that you would be able/allowed to share? What I've seen so far is that its length=

is 0x1c

Quoted 1 line

bytes on older controllers (e.g. Light Ridge, Port Ridge) and its content=

s are

Quoted 1 line

always the same, regardless if the controller is used in host mode or end=

point

Quoted 1 line

mode:

Sorry, currently nothing I can share.
=20

Quoted 4 lines

500: 0b 00 01 00 34 12 c1 01|50 09 00 00 39 00 00 00
510: 00 00 00 00 00 00 00 00 00 00 00 00
=20
However on Alpine Ridge the size of the VSEC has increased significantly =

to

Quoted 33 lines

0xd8 bytes, even though the version stayed at 1 as before:
=20
500: 0b 00 01 60 34 12 81 0d 50 09 b0 0c b9 06 18 08 510: 00 38 00 01 00 00 00 00 00 00 00 00 32 02 10 00 520: ef d3 00 40 00 00 00 00 f0 f0 30 c1 00 02 30 00 530: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 540: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 550: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 560: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 570: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 580: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 590: 00 00 00 08 00 00 00 00 00 00 00 00 00 00 00 00 5a0: 00 00 00 00 00 00 00 00 00 00 00 00 38 24 01 00 5b0: 08 40 00 1c 00 00 01 18 04 03 00 f0 06 00 00 00 5c0: 00 00 08 40 06 00 00 01 90 00 08 1b 00 08 80 08 5d0: 80 7f 08 00 00 00 00 00
=20
Thanks,
=20
Lukas
=20

Quoted 12 lines
Quoted 12 lines
drivers/pci/probe.c +10 -0
+	int vsec =3D 0;
+	u32 header;
+
+	while ((vsec =3D pci_find_next_ext_capability(dev, vsec,
+						    PCI_EXT_CAP_ID_VNDR))) {
+		pci_read_config_dword(dev, vsec + PCI_VNDR_HEADER,
 &header);
+
+		/* Is the device part of a Thunderbolt controller? */
+		if (dev->vendor =3D=3D PCI_VENDOR_ID_INTEL &&
+		    PCI_VNDR_HEADER_ID(header) =3D=3D PCI_VSEC_ID_INTEL_TBT)
Quoted 8 lines
Quoted 8 lines

drivers/pci/probe.c

Quoted 8 lines
drivers/pci/probe.c +6 -0
+			dev->is_thunderbolt =3D 1;
+	}
+
+	/*
+	 * Is the device attached with Thunderbolt?  Walk upwards and
+check

for

drivers/pci/probe.c +1 -0
+	 * each encountered bridge if it's part of a Thunderbolt controller=

.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
drivers/pci/probe.c +7 -0
+	 * Reaching the host bridge means dev is soldered to the mainboard.
+	 */
+	if (!dev->is_thunderbolt) {
+		struct pci_dev *parent =3D dev;
+
+		while ((parent =3D pci_upstream_bridge(parent)))
+			if (parent->is_thunderbolt) {
Quoted 8 lines
Quoted 8 lines

drivers/pci/probe.c void set_pcie_hotplug_bridge(struct pci_dev *pdev)

Quoted 8 lines
drivers/pci/probe.c +6 -0
+				dev->is_thunderbolt =3D 1;
+				break;
+			}
+	}
+}
+
 /**
  * pci_ext_cfg_is_aliased - is ext config space just an alias of std=

config?

Quoted 27 lines
Quoted 11 lines
Quoted 11 lines

* @dev: PCI device

drivers/pci/probe.c +3 -0
@@ -1358,6 +1389,9 @@ int pci_setup_device(struct pci_dev *dev)
 	/* need to have dev->class ready */
 	dev->cfg_size =3D pci_cfg_space_size(dev);
 
+	/* need to have dev->cfg_size ready */
+	set_pcie_vendor_specific(dev);
+
 	/* "Unknown power state" */
 	dev->current_state =3D PCI_UNKNOWN;
 
Quoted 10 lines

include/linux/pci.h struct pci_dev { · line 361

Quoted 10 lines
include/linux/pci.h index renamed from include/linux/pci.h +0 -0

e2d1a12..3c775e8
100644

include/linux/pci.h +1 -0
@@ -358,6 +358,7 @@ struct pci_dev {
 	unsigned int	is_virtfn:1;
 	unsigned int	reset_fn:1;
 	unsigned int    is_hotplug_bridge:1;
+	unsigned int	is_thunderbolt:1; /* part of Thunderbolt daisy chain

*/

Quoted 5 lines
Quoted 5 lines
include/linux/pci.h +0 -1
 	unsigned int    __aer_firmware_first_valid:1;
 	unsigned int	__aer_firmware_first:1;
 	unsigned int	broken_intx_masking:1;
--

2.11.0

#5 lore raw

PATCH v4 2/8 PCI: Allow runtime PM on Thunderbolt ports

In reply to Lukas Wunner

Currently PCIe ports are only allowed to go to D3 if the BIOS is dated 2015 or newer to avoid potential issues with old chipsets. However for Thunderbolt we know that even the oldest controller, Light Ridge (2010), is able to suspend its ports to D3 just fine.

We're about to add runtime PM for Thunderbolt on the Mac. Apple has released two EFI security updates in 2015 which encompass all machines with Thunderbolt, but the achieved power saving should be made available to users even if they haven't updated their BIOS. To this end, special-case Thunderbolt in pci_bridge_d3_possible().

This allows the Thunderbolt controller to power down but the root port to which the Thunderbolt controller is attached remains in D0 unless the EFI update is installed. Users can pass pcie_port_pm=force on the kernel command line if they cannot install the EFI update but still want to benefit from the additional power saving of putting the root port into D3. In practice, root ports can be suspended to D3 without issues at least on 2012 Ivy Bridge machines.

If the BIOS cut-off date is ever lowered to 2010, the Thunderbolt special case can be removed.

Cc Mika Westerberg <[email protected]>
Cc Rafael J. Wysocki <[email protected]>
Cc Andreas Noever <[email protected]>
Cc Tomas Winkler <[email protected]>
Cc Amir Levy <[email protected]>
---
 drivers/pci/pci.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)
drivers/pci/pci.c +6 -1
index a881c0d..8ed098d 100644
@@ -2224,7 +2224,7 @@ void pci_config_pm_runtime_put(struct pci_dev *pdev)
  * @bridge: Bridge to check
  *
  * This function checks if it is possible to move the bridge to D3.
- * Currently we only allow D3 for recent enough PCIe ports.
+ * Currently we only allow D3 for recent enough PCIe ports and Thunderbolt.
  */
 bool pci_bridge_d3_possible(struct pci_dev *bridge)
 {
@@ -2258,6 +2258,11 @@ bool pci_bridge_d3_possible(struct pci_dev *bridge)
 		    year >= 2015) {
 			return true;
 		}
+
+		/* Even the oldest 2010 Thunderbolt controller supports D3. */
+		if (bridge->is_thunderbolt)
+			return true;
+
 		break;
 	}
 
Signature · 1 line
2.11.0
Reviewed-by on v4 2/8
#6 lore raw

Re: [PATCH v4 2/8] PCI: Allow runtime PM on Thunderbolt ports

Quoted 19 lines · Lukas Wunner

Currently PCIe ports are only allowed to go to D3 if the BIOS is dated 2015 or newer to avoid potential issues with old chipsets. However for Thunderbolt we know that even the oldest controller, Light Ridge (2010), is able to suspend its ports to D3 just fine.

We're about to add runtime PM for Thunderbolt on the Mac. Apple has released two EFI security updates in 2015 which encompass all machines with Thunderbolt, but the achieved power saving should be made available to users even if they haven't updated their BIOS. To this end, special-case Thunderbolt in pci_bridge_d3_possible().

This allows the Thunderbolt controller to power down but the root port to which the Thunderbolt controller is attached remains in D0 unless the EFI update is installed. Users can pass pcie_port_pm=force on the kernel command line if they cannot install the EFI update but still want to benefit from the additional power saving of putting the root port into D3. In practice, root ports can be suspended to D3 without issues at least on 2012 Ivy Bridge machines.

Quoted 4 lines

If the BIOS cut-off date is ever lowered to 2010, the Thunderbolt special case can be removed.

Cc Mika Westerberg <[email protected]>
#7 lore raw

PATCH v4 3/8 PCI: Don't block runtime PM for Thunderbolt host hotplug ports

In reply to Lukas Wunner

Hotplug ports generally block their parents from suspending to D3hot as otherwise their interrupts couldn't be delivered.

An exception are Thunderbolt host controllers: They have a separate GPIO pin to side-band signal plug events even if the controller is powered down or its parent ports are suspended to D3. They can be told apart from Thunderbolt controllers in attached devices by checking if they're situated below a non-Thunderbolt device (typically a root port, or the downstream port of a PCIe switch in the case of the MacPro6,1).

To enable runtime PM for Thunderbolt on the Mac, the downstream bridges of a host controller must not block runtime PM on the upstream bridge as power to the chip is only cut once the upstream bridge has suspended. Amend the condition in pci_dev_check_d3cold() accordingly.

Cc Mika Westerberg <[email protected]>
Cc Rafael J. Wysocki <[email protected]>
Cc Andreas Noever <[email protected]>
Cc Tomas Winkler <[email protected]>
Cc Amir Levy <[email protected]>
---
 drivers/pci/pci.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)
drivers/pci/pci.c +12 -1
index 8ed098d..0b03fe7 100644
@@ -2271,6 +2271,7 @@ bool pci_bridge_d3_possible(struct pci_dev *bridge)
 
 static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
 {
+	struct pci_dev *parent, *grandparent;
 	bool *d3cold_ok = data;
 
 	if (/* The device needs to be allowed to go D3cold ... */
@@ -2284,7 +2285,17 @@ static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
 	    !pci_power_manageable(dev) ||
 
 	    /* Hotplug interrupts cannot be delivered if the link is down. */
-	    dev->is_hotplug_bridge)
+	    (dev->is_hotplug_bridge &&
+
+		/*
+		 * Exception:  Thunderbolt host controllers have a pin to
+		 * side-band signal plug events.  Their hotplug ports are
+		 * recognizable by having a non-Thunderbolt device as
+		 * grandparent.
+		 */
+		!(dev->is_thunderbolt && (parent = pci_upstream_bridge(dev)) &&
+				 (grandparent = pci_upstream_bridge(parent)) &&
+						!grandparent->is_thunderbolt)))
 
 		*d3cold_ok = false;
 
Signature · 1 line
2.11.0
#8 lore raw

Re: [PATCH v4 3/8] PCI: Don't block runtime PM for Thunderbolt host hotplug ports

Quoted 43 lines · Lukas Wunner

Hotplug ports generally block their parents from suspending to D3hot as otherwise their interrupts couldn't be delivered.

An exception are Thunderbolt host controllers: They have a separate GPIO pin to side-band signal plug events even if the controller is powered down or its parent ports are suspended to D3. They can be told apart from Thunderbolt controllers in attached devices by checking if they're situated below a non-Thunderbolt device (typically a root port, or the downstream port of a PCIe switch in the case of the MacPro6,1).

To enable runtime PM for Thunderbolt on the Mac, the downstream bridges of a host controller must not block runtime PM on the upstream bridge as power to the chip is only cut once the upstream bridge has suspended. Amend the condition in pci_dev_check_d3cold() accordingly.

Cc Mika Westerberg <[email protected]>
Cc Rafael J. Wysocki <[email protected]>
Cc Andreas Noever <[email protected]>
Cc Tomas Winkler <[email protected]>
Cc Amir Levy <[email protected]>
---
 drivers/pci/pci.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)
drivers/pci/pci.c +2 -1
index 8ed098d..0b03fe7 100644
@@ -2271,6 +2271,7 @@ bool pci_bridge_d3_possible(struct pci_dev *bridge)
 
 static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
 {
+	struct pci_dev *parent, *grandparent;
 	bool *d3cold_ok = data;
 
 	if (/* The device needs to be allowed to go D3cold ... */
@@ -2284,7 +2285,17 @@ static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
 	    !pci_power_manageable(dev) ||
 
 	    /* Hotplug interrupts cannot be delivered if the link is down. */
-	    dev->is_hotplug_bridge)
+	    (dev->is_hotplug_bridge &&

drivers/pci/pci.c static int pci_dev_check_d3cold(struct pci_dev *dev, void *data) · line 2298

Quoted 10 lines
drivers/pci/pci.c +10 -0
+
+		/*
+		 * Exception:  Thunderbolt host controllers have a pin to
+		 * side-band signal plug events.  Their hotplug ports are
+		 * recognizable by having a non-Thunderbolt device as
+		 * grandparent.
+		 */
+		!(dev->is_thunderbolt && (parent = pci_upstream_bridge(dev)) &&
+				 (grandparent = pci_upstream_bridge(parent)) &&
+						!grandparent->is_thunderbolt)))

Can you move this to its own helper function?

Quoted 5 lines
drivers/pci/pci.c +0 -0
 		*d3cold_ok = false;
 
Signature · 1 line
2.11.0
#9 lore raw

Re: [PATCH v4 3/8] PCI: Don't block runtime PM for Thunderbolt host hotplug ports

Quoted 41 lines · Mika Westerberg
Quoted 40 lines · Lukas Wunner

Hotplug ports generally block their parents from suspending to D3hot as otherwise their interrupts couldn't be delivered.

An exception are Thunderbolt host controllers: They have a separate GPIO pin to side-band signal plug events even if the controller is powered down or its parent ports are suspended to D3. They can be told apart from Thunderbolt controllers in attached devices by checking if they're situated below a non-Thunderbolt device (typically a root port, or the downstream port of a PCIe switch in the case of the MacPro6,1).

To enable runtime PM for Thunderbolt on the Mac, the downstream bridges of a host controller must not block runtime PM on the upstream bridge as power to the chip is only cut once the upstream bridge has suspended. Amend the condition in pci_dev_check_d3cold() accordingly.

Cc Mika Westerberg <[email protected]>
Cc Rafael J. Wysocki <[email protected]>
Cc Andreas Noever <[email protected]>
Cc Tomas Winkler <[email protected]>
Cc Amir Levy <[email protected]>
---
 drivers/pci/pci.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)
drivers/pci/pci.c +1 -0
index 8ed098d..0b03fe7 100644
@@ -2271,6 +2271,7 @@ bool pci_bridge_d3_possible(struct pci_dev *bridge)
 
 static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
 {
+	struct pci_dev *parent, *grandparent;
 	bool *d3cold_ok = data;
 
 	if (/* The device needs to be allowed to go D3cold ... */
@@ -2284,7 +2285,17 @@ static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
 	    !pci_power_manageable(dev) ||
 
Quoted 15 lines

drivers/pci/pci.c static int pci_dev_check_d3cold(struct pci_dev *dev, void *data) · line 2298

Quoted 13 lines
drivers/pci/pci.c +11 -1
 	    /* Hotplug interrupts cannot be delivered if the link is down. */
-	    dev->is_hotplug_bridge)
+	    (dev->is_hotplug_bridge &&
+
+		/*
+		 * Exception:  Thunderbolt host controllers have a pin to
+		 * side-band signal plug events.  Their hotplug ports are
+		 * recognizable by having a non-Thunderbolt device as
+		 * grandparent.
+		 */
+		!(dev->is_thunderbolt && (parent = pci_upstream_bridge(dev)) &&
+				 (grandparent = pci_upstream_bridge(parent)) &&
+						!grandparent->is_thunderbolt)))

Can you move this to its own helper function?

I can certainly do that.

Could one of you guys confirm that the code above is safe on non-Macs?

Specifically, the very first Thunderbolt chips (Light Ridge, Eagle Ridge) had no POC, i.e. they were unable to power themselves off. Apple put an ARM Cortex (NXP LPC1112) on the logic board which snoops on the connector lines for hotplug detection while the Thunderbolt controller is powered down. The power rails to the controller are brought up and down with separate load switches. This functionality became integrated into the controller starting with Cactus Ridge in 2012.

So I know the above code is safe on Macs. However on non-Macs these extra chips for power management may not exist, i.e. the controller stays on all the time and then I shouldn't suspend the upstream bridge to D3. I assume that such machines do not exist as Apple was pretty much the only vendor with Thunderbolt gear in the 2010-2012 time frame. The only other one I know of was the Sony Vaio Z21 which used the optical version of Thunderbolt to attach a docking station, but these are rare.

If you know of non-Macs which might be broken by the above code snippet, I could dmi-quirk this to Macs plus constrain to CONFIG_THUNDERBOLT being enabled.

Thanks,

Lukas

Quoted 6 lines
Quoted 6 lines
drivers/pci/pci.c +0 -0
 		*d3cold_ok = false;
 
Signature · 1 line
2.11.0
#10 lore raw

Re: [PATCH v4 3/8] PCI: Don't block runtime PM for Thunderbolt host hotplug ports

Quoted 79 lines · Lukas Wunner
Quoted 41 lines · Mika Westerberg
Quoted 40 lines · Lukas Wunner

Hotplug ports generally block their parents from suspending to D3hot as otherwise their interrupts couldn't be delivered.

An exception are Thunderbolt host controllers: They have a separate GPIO pin to side-band signal plug events even if the controller is powered down or its parent ports are suspended to D3. They can be told apart from Thunderbolt controllers in attached devices by checking if they're situated below a non-Thunderbolt device (typically a root port, or the downstream port of a PCIe switch in the case of the MacPro6,1).

To enable runtime PM for Thunderbolt on the Mac, the downstream bridges of a host controller must not block runtime PM on the upstream bridge as power to the chip is only cut once the upstream bridge has suspended. Amend the condition in pci_dev_check_d3cold() accordingly.

Cc Mika Westerberg <[email protected]>
Cc Rafael J. Wysocki <[email protected]>
Cc Andreas Noever <[email protected]>
Cc Tomas Winkler <[email protected]>
Cc Amir Levy <[email protected]>
---
 drivers/pci/pci.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)
drivers/pci/pci.c +1 -0
index 8ed098d..0b03fe7 100644
@@ -2271,6 +2271,7 @@ bool pci_bridge_d3_possible(struct pci_dev *bridge)
 
 static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
 {
+	struct pci_dev *parent, *grandparent;
 	bool *d3cold_ok = data;
 
 	if (/* The device needs to be allowed to go D3cold ... */
@@ -2284,7 +2285,17 @@ static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
 	    !pci_power_manageable(dev) ||
 
Quoted 15 lines

drivers/pci/pci.c static int pci_dev_check_d3cold(struct pci_dev *dev, void *data) · line 2298

Quoted 13 lines
drivers/pci/pci.c +11 -1
 	    /* Hotplug interrupts cannot be delivered if the link is down. */
-	    dev->is_hotplug_bridge)
+	    (dev->is_hotplug_bridge &&
+
+		/*
+		 * Exception:  Thunderbolt host controllers have a pin to
+		 * side-band signal plug events.  Their hotplug ports are
+		 * recognizable by having a non-Thunderbolt device as
+		 * grandparent.
+		 */
+		!(dev->is_thunderbolt && (parent = pci_upstream_bridge(dev)) &&
+				 (grandparent = pci_upstream_bridge(parent)) &&
+						!grandparent->is_thunderbolt)))

Can you move this to its own helper function?

I can certainly do that.

Could one of you guys confirm that the code above is safe on non-Macs?

Specifically, the very first Thunderbolt chips (Light Ridge, Eagle Ridge) had no POC, i.e. they were unable to power themselves off. Apple put an ARM Cortex (NXP LPC1112) on the logic board which snoops on the connector lines for hotplug detection while the Thunderbolt controller is powered down. The power rails to the controller are brought up and down with separate load switches. This functionality became integrated into the controller starting with Cactus Ridge in 2012.

So I know the above code is safe on Macs. However on non-Macs these extra chips for power management may not exist, i.e. the controller stays on all the time and then I shouldn't suspend the upstream bridge to D3. I assume that such machines do not exist as Apple was pretty much the only vendor with Thunderbolt gear in the 2010-2012 time frame. The only other one I know of was the Sony Vaio Z21 which used the optical version of Thunderbolt to attach a docking station, but these are rare.

Quoted 3 lines

If you know of non-Macs which might be broken by the above code snippet, I could dmi-quirk this to Macs plus constrain to CONFIG_THUNDERBOLT being enabled.

The thunderbolt chips I have seen all include the side-band hotplug detection GPIO. In addition the whole PCIe hierarchy is powered down when there is nothing connected. So in that sense, I don't see how this could break them.

Constraining this to CONFIG_THUNDERBOLT does not limit anything because distros will have it enabled anyway ;-) Having DMI quirk might be good idea, just in case.

#11 lore raw

Re: [PATCH v4 3/8] PCI: Don't block runtime PM for Thunderbolt host hotplug ports

Quoted 80 lines · Mika Westerberg
Quoted 79 lines · Lukas Wunner
Quoted 41 lines · Mika Westerberg
Quoted 40 lines · Lukas Wunner

Hotplug ports generally block their parents from suspending to D3hot as otherwise their interrupts couldn't be delivered.

An exception are Thunderbolt host controllers: They have a separate GPIO pin to side-band signal plug events even if the controller is powered down or its parent ports are suspended to D3. They can be told apart from Thunderbolt controllers in attached devices by checking if they're situated below a non-Thunderbolt device (typically a root port, or the downstream port of a PCIe switch in the case of the MacPro6,1).

To enable runtime PM for Thunderbolt on the Mac, the downstream bridges of a host controller must not block runtime PM on the upstream bridge as power to the chip is only cut once the upstream bridge has suspended. Amend the condition in pci_dev_check_d3cold() accordingly.

Cc Mika Westerberg <[email protected]>
Cc Rafael J. Wysocki <[email protected]>
Cc Andreas Noever <[email protected]>
Cc Tomas Winkler <[email protected]>
Cc Amir Levy <[email protected]>
---
 drivers/pci/pci.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)
drivers/pci/pci.c +1 -0
index 8ed098d..0b03fe7 100644
@@ -2271,6 +2271,7 @@ bool pci_bridge_d3_possible(struct pci_dev *bridge)
 
 static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
 {
+	struct pci_dev *parent, *grandparent;
 	bool *d3cold_ok = data;
 
 	if (/* The device needs to be allowed to go D3cold ... */
@@ -2284,7 +2285,17 @@ static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
 	    !pci_power_manageable(dev) ||
 
Quoted 15 lines

drivers/pci/pci.c static int pci_dev_check_d3cold(struct pci_dev *dev, void *data) · line 2298

Quoted 13 lines
drivers/pci/pci.c +11 -1
 	    /* Hotplug interrupts cannot be delivered if the link is down. */
-	    dev->is_hotplug_bridge)
+	    (dev->is_hotplug_bridge &&
+
+		/*
+		 * Exception:  Thunderbolt host controllers have a pin to
+		 * side-band signal plug events.  Their hotplug ports are
+		 * recognizable by having a non-Thunderbolt device as
+		 * grandparent.
+		 */
+		!(dev->is_thunderbolt && (parent = pci_upstream_bridge(dev)) &&
+				 (grandparent = pci_upstream_bridge(parent)) &&
+						!grandparent->is_thunderbolt)))

Can you move this to its own helper function?

I can certainly do that.

Could one of you guys confirm that the code above is safe on non-Macs?

Specifically, the very first Thunderbolt chips (Light Ridge, Eagle Ridge) had no POC, i.e. they were unable to power themselves off. Apple put an ARM Cortex (NXP LPC1112) on the logic board which snoops on the connector lines for hotplug detection while the Thunderbolt controller is powered down. The power rails to the controller are brought up and down with separate load switches. This functionality became integrated into the controller starting with Cactus Ridge in 2012.

So I know the above code is safe on Macs. However on non-Macs these extra chips for power management may not exist, i.e. the controller stays on all the time and then I shouldn't suspend the upstream bridge to D3. I assume that such machines do not exist as Apple was pretty much the only vendor with Thunderbolt gear in the 2010-2012 time frame. The only other one I know of was the Sony Vaio Z21 which used the optical version of Thunderbolt to attach a docking station, but these are rare.

Quoted 8 lines
Quoted 3 lines

If you know of non-Macs which might be broken by the above code snippet, I could dmi-quirk this to Macs plus constrain to CONFIG_THUNDERBOLT being enabled.

The thunderbolt chips I have seen all include the side-band hotplug detection GPIO. In addition the whole PCIe hierarchy is powered down when there is nothing connected. So in that sense, I don't see how this could break them.

The pin to side-band signal a plug event is not present on Light Ridge (and presumably Light Peak), only on Eagle Ridge and newer (2011+).

The pins for power control (Force Power, Go2Sx, Ok2Go2Sx) are not present on Light Ridge and Eagle Ridge, only on Cactus Ridge and newer (2012+).

In other words, Thunderbolt controllers had no power control built in before Cactus Ridge. If you've only seen chips which have the plug event pin then you've only seen newer chips. ;-)

Quoted 3 lines

Constraining this to CONFIG_THUNDERBOLT does not limit anything because distros will have it enabled anyway ;-) Having DMI quirk might be good idea, just in case.

Actually, never mind. It occurred to me that we only allow hotplug ports to suspend in pci_bridge_d3_possible() if they're handled natively by the OS, which is only the case on Macs. On non-Macs the hotplug ports block their parent ports from suspending by virtue of staying in D0 all the time. Thus it's meaningless whether or not they block their parent ports from suspending in pci_dev_check_d3cold().

So the patch is safe, I just forgot all this context because I wrote it months ago. I've now amended the commit message with this background information.

Thanks,

Lukas

#12 lore raw

PATCH v4 4/8 Revert "PM / Runtime: Remove the exported function pm_children_suspended()"

In reply to Lukas Wunner

This reverts commit 62006c1702b3b1be0c0726949e0ee0ea2326be9c which removed pm_children_suspended() because it had only a single caller. We're about to add a second caller, so establish the status quo ante.

Cc Ulf Hansson <[email protected]>
Cc Rafael J. Wysocki <[email protected]>
---
 drivers/base/power/runtime.c | 3 +--
 include/linux/pm_runtime.h   | 7 +++++++
 2 files changed, 8 insertions(+), 2 deletions(-)
drivers/base/power/runtime.c +1 -2
index 872eac4..03293c3 100644
@@ -243,8 +243,7 @@ static int rpm_check_suspend_allowed(struct device *dev)
 		retval = -EACCES;
 	else if (atomic_read(&dev->power.usage_count) > 0)
 		retval = -EAGAIN;
-	else if (!dev->power.ignore_children &&
-			atomic_read(&dev->power.child_count))
+	else if (!pm_children_suspended(dev))
 		retval = -EBUSY;
 
 	/* Pending resume requests take precedence over suspends. */
include/linux/pm_runtime.h +7 -0
index ca4823e..7de2aa5c 100644
@@ -66,6 +66,12 @@ static inline void pm_suspend_ignore_children(struct device *dev, bool enable)
 	dev->power.ignore_children = enable;
 }
 
+static inline bool pm_children_suspended(struct device *dev)
+{
+	return dev->power.ignore_children
+		|| !atomic_read(&dev->power.child_count);
+}
+
 static inline void pm_runtime_get_noresume(struct device *dev)
 {
 	atomic_inc(&dev->power.usage_count);
@@ -161,6 +167,7 @@ static inline void pm_runtime_allow(struct device *dev) {}
 static inline void pm_runtime_forbid(struct device *dev) {}
 
 static inline void pm_suspend_ignore_children(struct device *dev, bool enable) {}
+static inline bool pm_children_suspended(struct device *dev) { return false; }
 static inline void pm_runtime_get_noresume(struct device *dev) {}
 static inline void pm_runtime_put_noidle(struct device *dev) {}
 static inline bool device_run_wake(struct device *dev) { return false; }
Signature · 1 line
2.11.0
#13 lore raw

PATCH v4 5/8 PM: Make requirements of dev_pm_domain_set() more precise

In reply to Lukas Wunner

Since commit 989561de9b51 ("PM / Domains: add setter for dev.pm_domain") a PM domain may only be assigned to unbound devices.

The motivation was not made explicit in the changelog other than "in the general case that can cause problems and also [...] we can simplify code quite a bit if we can always assume that". Rafael J. Wysocki elaborated in a mailing list conversation that "setting a PM domain generally changes the set of PM callbacks for the device and it may not be safe to call it after the driver has been bound".

The concern seems to be that if a device is put to sleep and its PM callbacks are changed, the device may end up in an undefined state or not resume at all. The real underlying requirement is thus to ensure that the device is awake and execution of its PM callbacks is prevented while the PM domain is assigned. Unbound devices happen to fulfill this requirement, but bound devices can be made to satisfy it as well:
The caller can prevent execution of PM ops with lock_system_sleep() and by holding a runtime PM reference to the device.

Accordingly, adjust dev_pm_domain_set() to WARN only if the device is in the midst of a system sleep transition, or runtime PM is enabled and the device is either not active or may become inactive imminently (because it has no active children or its refcount is zero).

The change is required to support runtime PM for Thunderbolt on the Mac, which poses the unique issue that a child device (the NHI) needs to assign a PM domain to its grandparent (the upstream bridge). Because the grandparent's driver is built-in and the child's driver is a module, the grandparent is usually already bound when the child probes, resulting in a WARN splat when calling dev_pm_domain_set(). However the PM core guarantees both that the grandparent is active and that system sleep is not commenced until the child has finished probing. So in this case it is safe to call dev_pm_domain_set() from the child's ->probe hook and the WARN splat is entirely gratuitous.

Note that commit e79aee49bcf9 ("PM: Avoid false-positive warnings in dev_pm_domain_set()") modified the WARN to not apply if a PM domain is removed. This is unsafe as it allows removal of the PM domain while the device is asleep. The present commit rectifies this.

Cc Ulf Hansson <[email protected]>
Cc Tomeu Vizoso <[email protected]>
Cc Rafael J. Wysocki <[email protected]>
---
 drivers/base/power/common.c | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)
drivers/base/power/common.c +11 -4
index f6a9ad5..d02c1e0 100644
@@ -13,6 +13,7 @@
 #include <linux/pm_clock.h>
 #include <linux/acpi.h>
 #include <linux/pm_domain.h>
+#include <linux/pm_runtime.h>
 
 #include "power.h"
 
@@ -136,8 +137,10 @@ void dev_pm_domain_detach(struct device *dev, bool power_off)
  * @dev: Device whose PM domain is to be set.
  * @pd: PM domain to be set, or NULL.
  *
- * Sets the PM domain the device belongs to. The PM domain of a device needs
- * to be set before its probe finishes (it's bound to a driver).
+ * Sets the PM domain the device belongs to.  The PM domain of a device needs
+ * to be set while the device is awake.  This is guaranteed during ->probe.
+ * Otherwise the caller is responsible for ensuring wakefulness, e.g. by
+ * holding a runtime PM reference as well as invoking lock_system_sleep().
  *
  * This function must be called with the device lock held.
  */
@@ -146,8 +149,12 @@ void dev_pm_domain_set(struct device *dev, struct dev_pm_domain *pd)
 	if (dev->pm_domain == pd)
 		return;
 
-	WARN(pd && device_is_bound(dev),
-	     "PM domains can only be changed for unbound devices\n");
+	WARN(dev->power.is_prepared || dev->power.is_suspended ||
+	     (pm_runtime_enabled(dev) &&
+	      (dev->power.runtime_status != RPM_ACTIVE ||
+	       (pm_children_suspended(dev) &&
+		!atomic_read(&dev->power.usage_count)))),
+	     "PM domains can only be changed for awake devices\n");
 	dev->pm_domain = pd;
 	device_pm_check_callbacks(dev);
 }
Signature · 1 line
2.11.0
#14 lore raw

PATCH v4 6/8 PM / sleep: Define constant for direct_complete

In reply to Lukas Wunner

From: Chen Yu <[email protected]>

The PM core introduced the ability to keep devices runtime suspended during the entire system sleep process with commit aae4518b3124 ("PM / sleep: Mechanism to avoid resuming runtime-suspended devices unnecessarily").

Drivers opt in to this so-called "direct_complete" mechanism by returning a positive value from their ->prepare hook. Usually this is achieved with a "return 1;" statement, which looks somewhat cryptic to readers not intimately familiar with the PM core.

Improve clarity by defining a DPM_DIRECT_COMPLETE constant which drivers may use instead.

Cc Lee Jones <[email protected]>
[LW: separate this hunk out of mfd patch, add commit message]
---
 include/linux/pm.h | 7 +++++++
 1 file changed, 7 insertions(+)
include/linux/pm.h +7 -0
index f926af4..2d651a3 100644
@@ -788,4 +788,11 @@ enum dpm_order {
 	DPM_ORDER_DEV_LAST,
 };
 
+/*
+ * Return this from system suspend/hibernation ->prepare() callback to
+ * request the core to leave the device runtime-suspended during system
+ * suspend if possible.
+ */
+#define DPM_DIRECT_COMPLETE 1
+
 #endif /* _LINUX_PM_H */
Signature · 1 line
2.11.0
#15 lore raw

PATCH v4 7/8 thunderbolt: Power down controller when idle

In reply to Lukas Wunner

Document and implement Apple's ACPI-based (but nonstandard) pm mechanism for Thunderbolt. Briefly, an ACPI method provided by Apple is used to cut power to the controller. A GPE is enabled while the controller is powered down which sideband-signals a plug event, whereupon we reinstate power using the ACPI method.

This saves 1.7 W on machines with a Light Ridge controller and is reported to save 4 W on Cactus Ridge 4C and Falcon Ridge 4C. (I believe 4 W includes the bus power drawn by Apple's Gigabit Ethernet adapter.) It fixes (at least partially) a power regression introduced in 3.17 by commit 7bc5a2bad0b8 ("ACPI: Support _OSI("Darwin") correctly").

A Thunderbolt controller appears to the OS as a set of virtual devices:
One upstream bridge, multiple downstream bridges and one NHI (Native Host Interface). The upstream and downstream bridges represent a PCIe switch (see definition of a switch in the PCIe spec). The NHI device is used to manage the switch fabric. Hotplugged devices appear behind the downstream bridges:

(Root Port) ---- Upstream Bridge --+-- Downstream Bridge 0 ---- NHI

                                     +-- Downstream Bridge 1 --
                                     +-- Downstream Bridge 2 --
                                     ...

Power is cut to the entire set of devices. The Linux pm model is hierarchical and assumes that a child cannot resume before its parent. To conform to this model, power control must be governed by the Thunderbolt controller's topmost device, which is the upstream bridge. The NHI and downstream bridges go to D3hot independently and the upstream bridge goes to D3cold once all its children have suspended. This commit only adds runtime pm for the upstream bridge. Runtime pm for the NHI is added in a separate commit to signify its independence. Runtime pm for the downstream bridges is handled by the pcieport driver.

Because Apple's ACPI methods are nonstandard, a struct dev_pm_domain is used to override the PCI bus pm_ops. The thunderbolt driver binds to the NHI, thus the dev_pm_domain is assigned to the upstream bridge when its grandchild ->probes and evicted when it ->removes.

There are no Thunderbolt specs publicly available from Intel or Apple, so I've included documentation to the extent that I was able to reverse- engineer things. Documentation on the Go2Sx and Ok2Go2Sx pins is tentative as those are missing on my Light Ridge. Apple only uses them on Cactus Ridge 4C. Someone with such a controller needs to find out through experimentation if the documentation is accurate and amend it if necessary.

To maximize power saving, the controller utilizes the PM core's direct- complete procedure, i.e. it stays suspended during the system sleep process.

Bugzilla: https://bugzilla.kernel.org/show_bug.cgi?id=92111

Cc Andreas Noever <[email protected]>
---
 drivers/thunderbolt/Kconfig  |   3 +-
 drivers/thunderbolt/Makefile |   4 +-
 drivers/thunderbolt/nhi.c    |   3 +
 drivers/thunderbolt/power.c  | 346 +++++++++++++++++++++++++++++++++++++++++++
 drivers/thunderbolt/power.h  |  37 +++++
 drivers/thunderbolt/tb.h     |   2 +
 6 files changed, 392 insertions(+), 3 deletions(-)
 create mode 100644 drivers/thunderbolt/power.c
 create mode 100644 drivers/thunderbolt/power.h
drivers/thunderbolt/Kconfig +2 -1
index d35db16..41625cf 100644
@@ -1,9 +1,10 @@
 menuconfig THUNDERBOLT
 	tristate "Thunderbolt support for Apple devices"
-	depends on PCI
+	depends on PCI && ACPI
 	depends on X86 || COMPILE_TEST
 	select APPLE_PROPERTIES if EFI_STUB && X86
 	select CRC32
+	select PM
 	help
 	  Cactus Ridge Thunderbolt Controller driver
 	  This driver is required if you want to hotplug Thunderbolt devices on
drivers/thunderbolt/Makefile +2 -2
index 5d1053c..b220825 100644
@@ -1,3 +1,3 @@
 obj-${CONFIG_THUNDERBOLT} := thunderbolt.o
-thunderbolt-objs := nhi.o ctl.o tb.o switch.o cap.o path.o tunnel_pci.o eeprom.o
-
+thunderbolt-objs := nhi.o ctl.o tb.o switch.o cap.o path.o tunnel_pci.o \
+		    eeprom.o power.o
drivers/thunderbolt/nhi.c +3 -0
index a8c2041..88fb2fb 100644
@@ -605,6 +605,8 @@ static int nhi_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	}
 	pci_set_drvdata(pdev, tb);
 
+	thunderbolt_power_init(tb);
+
 	return 0;
 }
 
@@ -612,6 +614,7 @@ static void nhi_remove(struct pci_dev *pdev)
 {
 	struct tb *tb = pci_get_drvdata(pdev);
 	struct tb_nhi *nhi = tb->nhi;
+	thunderbolt_power_fini(tb);
 	thunderbolt_shutdown_and_free(tb);
 	nhi_shutdown(nhi);
 }
drivers/thunderbolt/power.c new file +346 -0

346 lines not on this page · raw message

drivers/thunderbolt/power.h new file +37 -0
new file mode 100644
index 0000000..4ab9b29
@@ -0,0 +1,37 @@
+/*
+ * power.h - power thunderbolt controller down when idle
+ * Copyright (C) 2016 Lukas Wunner <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License (version 2) as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef THUNDERBOLT_POWER_H
+#define THUNDERBOLT_POWER_H
+
+#include <linux/acpi.h>
+#include <linux/pm_domain.h>
+
+#include "tb.h"
+
+struct tb_power {
+	struct tb *tb;
+	struct dev_pm_domain pm_domain; /* assigned to upstream bridge */
+	unsigned long long wake_gpe; /* hotplug interrupt during powerdown */
+	acpi_handle set; /* method to power controller up/down */
+	acpi_handle get; /* method to query power state */
+};
+
+void thunderbolt_power_init(struct tb *tb);
+void thunderbolt_power_fini(struct tb *tb);
+
+#endif
drivers/thunderbolt/tb.h +2 -0
index 61d57ba..43aed58 100644
@@ -11,6 +11,7 @@
 
 #include "tb_regs.h"
 #include "ctl.h"
+#include "power.h"
 
 /**
  * struct tb_switch - a thunderbolt switch
@@ -103,6 +104,7 @@ struct tb {
 				 */
 	struct tb_nhi *nhi;
 	struct tb_ctl *ctl;
+	struct tb_power *power;
 	struct workqueue_struct *wq; /* ordered workqueue for plug events */
 	struct tb_switch *root_switch;
 	struct list_head tunnel_list; /* list of active PCIe tunnels */
Signature · 1 line
2.11.0
#16 lore raw

PATCH v4 8/8 thunderbolt: Runtime suspend NHI when idle

In reply to Lukas Wunner

Runtime suspend the NHI when no Thunderbolt devices have been plugged in for 10 sec (user-configurable via autosuspend_delay_ms in sysfs).

The NHI is not able to detect plug events while suspended, it relies on the GPE handler to resume it on hotplug.

After the NHI resumes, it takes about 700 ms until a hotplug event appears on the RX ring. In case autosuspend_delay_ms has been reduced to 0 by the user, we need to wait in tb_resume() to avoid going back to sleep before we had a chance to detect a hotplugged device. A runtime pm ref is held for the duration of tb_handle_hotplug() to keep the NHI awake while the hotplug event is processed.

Apart from that we acquire a runtime pm ref for each newly allocated switch (except for the root switch) and drop one when a switch is freed, thereby ensuring the NHI stays active as long as devices are plugged in. This behaviour is identical to the macOS driver.

Cc Andreas Noever <[email protected]>
---
 drivers/thunderbolt/nhi.c    |  2 ++
 drivers/thunderbolt/power.c  |  9 +++++++++
 drivers/thunderbolt/switch.c |  9 +++++++++
 drivers/thunderbolt/tb.c     | 13 +++++++++++++
 4 files changed, 33 insertions(+)
drivers/thunderbolt/nhi.c +2 -0
index 88fb2fb..319ed81 100644
@@ -632,6 +632,8 @@ static void nhi_remove(struct pci_dev *pdev)
 					    * pci-tunnels stay alive.
 					    */
 	.restore_noirq = nhi_resume_noirq,
+	.runtime_suspend = nhi_suspend_noirq,
+	.runtime_resume = nhi_resume_noirq,
 };
 
 static struct pci_device_id nhi_ids[] = {
drivers/thunderbolt/power.c +9 -0
index 6e7ef07..06f535c 100644
@@ -319,6 +319,12 @@ void thunderbolt_power_init(struct tb *tb)
 
 	tb->power = power;
 
+	pm_runtime_allow(nhi_dev);
+	pm_runtime_set_autosuspend_delay(nhi_dev, 10000);
+	pm_runtime_use_autosuspend(nhi_dev);
+	pm_runtime_mark_last_busy(nhi_dev);
+	pm_runtime_put_autosuspend(nhi_dev);
+
 	return;
 
 err_free:
@@ -335,6 +341,9 @@ void thunderbolt_power_fini(struct tb *tb)
 	if (!power)
 		return; /* thunderbolt_power_init() failed */
 
+	pm_runtime_get(nhi_dev);
+	pm_runtime_forbid(nhi_dev);
+
 	tb->power = NULL;
 	dev_pm_domain_set(upstream_dev, NULL);
 
drivers/thunderbolt/switch.c +9 -0
index c6f30b1..422fe6e 100644
@@ -5,6 +5,7 @@
  */
 
 #include <linux/delay.h>
+#include <linux/pm_runtime.h>
 #include <linux/slab.h>
 
 #include "tb.h"
@@ -326,6 +327,11 @@ void tb_switch_free(struct tb_switch *sw)
 	if (!sw->is_unplugged)
 		tb_plug_events_active(sw, false);
 
+	if (sw != sw->tb->root_switch) {
+		pm_runtime_mark_last_busy(&sw->tb->nhi->pdev->dev);
+		pm_runtime_put_autosuspend(&sw->tb->nhi->pdev->dev);
+	}
+
 	kfree(sw->ports);
 	kfree(sw->drom);
 	kfree(sw);
@@ -420,6 +426,9 @@ struct tb_switch *tb_switch_alloc(struct tb *tb, u64 route)
 	if (tb_plug_events_active(sw, true))
 		goto err;
 
+	if (tb->root_switch)
+		pm_runtime_get(&tb->nhi->pdev->dev);
+
 	return sw;
 err:
 	kfree(sw->ports);
drivers/thunderbolt/tb.c +13 -0
index 24b6d30..a3fedf9 100644
@@ -7,6 +7,7 @@
 #include <linux/slab.h>
 #include <linux/errno.h>
 #include <linux/delay.h>
+#include <linux/pm_runtime.h>
 
 #include "tb.h"
 #include "tb_regs.h"
@@ -217,8 +218,11 @@ static void tb_handle_hotplug(struct work_struct *work)
 {
 	struct tb_hotplug_event *ev = container_of(work, typeof(*ev), work);
 	struct tb *tb = ev->tb;
+	struct device *dev = &tb->nhi->pdev->dev;
 	struct tb_switch *sw;
 	struct tb_port *port;
+
+	pm_runtime_get(dev);
 	mutex_lock(&tb->lock);
 	if (!tb->hotplug_active)
 		goto out; /* during init, suspend or shutdown */
@@ -274,6 +278,8 @@ static void tb_handle_hotplug(struct work_struct *work)
 out:
 	mutex_unlock(&tb->lock);
 	kfree(ev);
+	pm_runtime_mark_last_busy(dev);
+	pm_runtime_put_autosuspend(dev);
 }
 
 /**
@@ -433,4 +439,11 @@ void thunderbolt_resume(struct tb *tb)
 	tb->hotplug_active = true;
 	mutex_unlock(&tb->lock);
 	tb_info(tb, "resume finished\n");
+
+	/*
+	 * If runtime resuming due to a hotplug event (rather than resuming
+	 * from system sleep), wait for it to arrive. May take about 700 ms.
+	 */
+	if (tb->nhi->pdev->dev.power.runtime_status == RPM_RESUMING)
+		msleep(1000);
 }
Signature · 1 line
2.11.0