Skip to contentSkip to conversation

PCI: Add a mutex to protect the global list pci_domain_busn_res_list

PATCH In review Dexuan Cui 4 participants 5 messages Apr 19, 2024 – Apr 29, 2024

Subsystems (3)
lore
Share & tools
lore

Dexuan Cui

author maintainer
#0 lore raw

PATCH 1/1 PCI: Add a mutex to protect the global list pci_domain_busn_res_list

There has been an effort to make the pci-hyperv driver support async-probing to reduce the boot time. With async-probing, multiple kernel threads can be running hv_pci_probe() -> create_root_hv_pci_bus() -> pci_scan_root_bus_bridge() -> pci_bus_insert_busn_res() at the same time to update the global list, causing list corruption.

Add a mutex to protect the list.

---
 drivers/pci/probe.c | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)
drivers/pci/probe.c +18 -7
index e19b79821dd6..1327fd820b24 100644
@@ -37,6 +37,7 @@ LIST_HEAD(pci_root_buses);
 EXPORT_SYMBOL(pci_root_buses);
 
 static LIST_HEAD(pci_domain_busn_res_list);
+static DEFINE_MUTEX(pci_domain_busn_res_list_lock);
 
 struct pci_domain_busn_res {
 	struct list_head list;
@@ -47,14 +48,22 @@ struct pci_domain_busn_res {
 static struct resource *get_pci_domain_busn_res(int domain_nr)
 {
 	struct pci_domain_busn_res *r;
+	struct resource *ret;
 
-	list_for_each_entry(r, &pci_domain_busn_res_list, list)
-		if (r->domain_nr == domain_nr)
-			return &r->res;
+	mutex_lock(&pci_domain_busn_res_list_lock);
+
+	list_for_each_entry(r, &pci_domain_busn_res_list, list) {
+		if (r->domain_nr == domain_nr) {
+			ret = &r->res;
+			goto out;
+		}
+	}
 
 	r = kzalloc(sizeof(*r), GFP_KERNEL);
-	if (!r)
-		return NULL;
+	if (!r) {
+		ret = NULL;
+		goto out;
+	}
 
 	r->domain_nr = domain_nr;
 	r->res.start = 0;
@@ -62,8 +71,10 @@ static struct resource *get_pci_domain_busn_res(int domain_nr)
 	r->res.flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED;
 
 	list_add_tail(&r->list, &pci_domain_busn_res_list);
-
-	return &r->res;
+	ret = &r->res;
+out:
+	mutex_unlock(&pci_domain_busn_res_list_lock);
+	return ret;
 }
 
 /*
Signature · 1 line
2.25.1
maintainer
#1 lore raw
Quoted 72 lines

-----Original Message-----
From: Dexuan Cui <[email protected]>
Sent: Thursday, April 18, 2024 9:53 PM
To: [email protected]; [email protected]; KY Srinivasan <[email protected]>; Haiyang Zhang <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; [email protected]; Boqun Feng <[email protected]>; Sunil Muthuswamy <[email protected]>; Saurabh Singh Sengar <[email protected]>; Dexuan Cui <[email protected]>
Subject: [PATCH] PCI: Add a mutex to protect the global list pci_domain_busn_res_list

There has been an effort to make the pci-hyperv driver support async-probing to reduce the boot time. With async-probing, multiple kernel threads can be running hv_pci_probe() -> create_root_hv_pci_bus() ->
pci_scan_root_bus_bridge() -> pci_bus_insert_busn_res() at the same time to
update the global list, causing list corruption.

Add a mutex to protect the list.

---
 drivers/pci/probe.c | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)
drivers/pci/probe.c +14 -5
index e19b79821dd6..1327fd820b24 100644
@@ -37,6 +37,7 @@ LIST_HEAD(pci_root_buses);
 EXPORT_SYMBOL(pci_root_buses);
 
 static LIST_HEAD(pci_domain_busn_res_list);
+static DEFINE_MUTEX(pci_domain_busn_res_list_lock);
 
 struct pci_domain_busn_res {
 	struct list_head list;
@@ -47,14 +48,22 @@ struct pci_domain_busn_res {
 static struct resource *get_pci_domain_busn_res(int domain_nr)
 {
 	struct pci_domain_busn_res *r;
+	struct resource *ret;
 
-	list_for_each_entry(r, &pci_domain_busn_res_list, list)
-		if (r->domain_nr == domain_nr)
-			return &r->res;
+	mutex_lock(&pci_domain_busn_res_list_lock);
+
+	list_for_each_entry(r, &pci_domain_busn_res_list, list) {
+		if (r->domain_nr == domain_nr) {
+			ret = &r->res;
+			goto out;
+		}
+	}
 
 	r = kzalloc(sizeof(*r), GFP_KERNEL);
-	if (!r)
-		return NULL;
+	if (!r) {
+		ret = NULL;
+		goto out;
+	}
 
 	r->domain_nr = domain_nr;
 	r->res.start = 0;
@@ -62,8 +71,10 @@ static struct resource *get_pci_domain_busn_res(int
 domain_nr)

drivers/pci/probe.c static struct resource *get_pci_domain_busn_res(int · line 79

Quoted 10 lines
drivers/pci/probe.c +4 -2
 	r->res.flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED;
 
 	list_add_tail(&r->list, &pci_domain_busn_res_list);
-
-	return &r->res;
+	ret = &r->res;
+out:
+	mutex_unlock(&pci_domain_busn_res_list_lock);
+	return ret;
 }

The patch is for common pci code. So, this bug has been there for a while? Do you have a sample stack trace of the crash?

I checked pci-hyperv, it doesn't define the .driver.probe_type, so PROBE_DEFAULT_STRATEGY is in effect. driver_allows_async_probing() returns false unless kernel/mod param requests async. So async probing haven't been practiced here.

If in the future, we change the pci-hyperv's probe_type to PROBE_PREFER_ASYNCHRONOUS, how does it affect the underlying PCI device's probes within the same device type?
For example, MANA driver doesn't set probe_type. Will pci-hyperv's async probing cause async probing or potentially nondeterministic naming for MANA devices?

Thanks,
- Haiyang

#2 lore raw
In reply to Dexuan Cui
Quoted 26 lines · Dexuan Cui

There has been an effort to make the pci-hyperv driver support async-probing to reduce the boot time. With async-probing, multiple kernel threads can be running hv_pci_probe() -> create_root_hv_pci_bus() -> pci_scan_root_bus_bridge() -> pci_bus_insert_busn_res() at the same time to update the global list, causing list corruption.

Add a mutex to protect the list.

---
 drivers/pci/probe.c | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)
drivers/pci/probe.c +1 -0
index e19b79821dd6..1327fd820b24 100644
@@ -37,6 +37,7 @@ LIST_HEAD(pci_root_buses);
 EXPORT_SYMBOL(pci_root_buses);
 
 static LIST_HEAD(pci_domain_busn_res_list);
+static DEFINE_MUTEX(pci_domain_busn_res_list_lock);
 
 struct pci_domain_busn_res {
 	struct list_head list;

drivers/pci/probe.c struct pci_domain_busn_res { · line 53

Quoted 9 lines
drivers/pci/probe.c +2 -3
@@ -47,14 +48,22 @@ struct pci_domain_busn_res {
 static struct resource *get_pci_domain_busn_res(int domain_nr)
 {
 	struct pci_domain_busn_res *r;
+	struct resource *ret;
 
-	list_for_each_entry(r, &pci_domain_busn_res_list, list)
-		if (r->domain_nr == domain_nr)
-			return &r->res;
+	mutex_lock(&pci_domain_busn_res_list_lock);

Using

	guard(mutex)(&pci_domain_busn_res_list_lock);

to simple logic, especially there are goto.

You can avoid goto out, direct return NULL;

Frank

Quoted 34 lines
drivers/pci/probe.c +15 -4
+
+	list_for_each_entry(r, &pci_domain_busn_res_list, list) {
+		if (r->domain_nr == domain_nr) {
+			ret = &r->res;
+			goto out;
+		}
+	}
 
 	r = kzalloc(sizeof(*r), GFP_KERNEL);
-	if (!r)
-		return NULL;
+	if (!r) {
+		ret = NULL;
+		goto out;
+	}
 
 	r->domain_nr = domain_nr;
 	r->res.start = 0;
@@ -62,8 +71,10 @@ static struct resource *get_pci_domain_busn_res(int domain_nr)
 	r->res.flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED;
 
 	list_add_tail(&r->list, &pci_domain_busn_res_list);
-
-	return &r->res;
+	ret = &r->res;
+out:
+	mutex_unlock(&pci_domain_busn_res_list_lock);
+	return ret;
 }
 
 /*
Signature · 1 line
2.25.1
#3 lore raw
In reply to Dexuan Cui
Quoted 7 lines · Dexuan Cui

There has been an effort to make the pci-hyperv driver support async-probing to reduce the boot time. With async-probing, multiple kernel threads can be running hv_pci_probe() -> create_root_hv_pci_bus() -> pci_scan_root_bus_bridge() -> pci_bus_insert_busn_res() at the same time to update the global list, causing list corruption.

Add a mutex to protect the list.

I think it's a good idea to support probing multiple PCI root buses in parallel.

The problem in get_pci_domain_busn_res() is the global pci_domain_busn_res_list. I'm not even sure what that list contains, since it's a lookup by "domain_nr". In the hv case, you probably have one host bridge per domain, but in general there may be multiple root buses in the same domain, e.g.,

ACPI: PCI Root Bridge [PC00] (domain 0000 [bus 00-16])
ACPI: PCI Root Bridge [PC01] (domain 0000 [bus 17-39])
ACPI: PCI Root Bridge [PC02] (domain 0000 [bus 3a-5c])
...

We only use get_pci_domain_busn_res() for root buses, and we should know the bus number range for root buses when we set up the struct pci_host_bridge, so it seems like we should keep the bus number resource there instead of allocating it in this sort of random place.

Then we shouldn't need this weird pci_domain_busn_res_list at all.

Quoted 61 lines
---
 drivers/pci/probe.c | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)
drivers/pci/probe.c +18 -7
index e19b79821dd6..1327fd820b24 100644
@@ -37,6 +37,7 @@ LIST_HEAD(pci_root_buses);
 EXPORT_SYMBOL(pci_root_buses);
 
 static LIST_HEAD(pci_domain_busn_res_list);
+static DEFINE_MUTEX(pci_domain_busn_res_list_lock);
 
 struct pci_domain_busn_res {
 	struct list_head list;
@@ -47,14 +48,22 @@ struct pci_domain_busn_res {
 static struct resource *get_pci_domain_busn_res(int domain_nr)
 {
 	struct pci_domain_busn_res *r;
+	struct resource *ret;
 
-	list_for_each_entry(r, &pci_domain_busn_res_list, list)
-		if (r->domain_nr == domain_nr)
-			return &r->res;
+	mutex_lock(&pci_domain_busn_res_list_lock);
+
+	list_for_each_entry(r, &pci_domain_busn_res_list, list) {
+		if (r->domain_nr == domain_nr) {
+			ret = &r->res;
+			goto out;
+		}
+	}
 
 	r = kzalloc(sizeof(*r), GFP_KERNEL);
-	if (!r)
-		return NULL;
+	if (!r) {
+		ret = NULL;
+		goto out;
+	}
 
 	r->domain_nr = domain_nr;
 	r->res.start = 0;
@@ -62,8 +71,10 @@ static struct resource *get_pci_domain_busn_res(int domain_nr)
 	r->res.flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED;
 
 	list_add_tail(&r->list, &pci_domain_busn_res_list);
-
-	return &r->res;
+	ret = &r->res;
+out:
+	mutex_unlock(&pci_domain_busn_res_list_lock);
+	return ret;
 }
 
 /*
Signature · 1 line
2.25.1
#4 lore raw
Quoted 23 lines · Bjorn Helgaas
Quoted 7 lines · Dexuan Cui

There has been an effort to make the pci-hyperv driver support async-probing to reduce the boot time. With async-probing, multiple kernel threads can be running hv_pci_probe() -> create_root_hv_pci_bus() -> pci_scan_root_bus_bridge() -> pci_bus_insert_busn_res() at the same time to update the global list, causing list corruption.

Add a mutex to protect the list.

I think it's a good idea to support probing multiple PCI root buses in parallel.

The problem in get_pci_domain_busn_res() is the global pci_domain_busn_res_list. I'm not even sure what that list contains, since it's a lookup by "domain_nr". In the hv case, you probably have one host bridge per domain, but in general there may be multiple root buses in the same domain, e.g.,

ACPI: PCI Root Bridge [PC00] (domain 0000 [bus 00-16])
ACPI: PCI Root Bridge [PC01] (domain 0000 [bus 17-39])
ACPI: PCI Root Bridge [PC02] (domain 0000 [bus 3a-5c])
...

Quoted 6 lines

We only use get_pci_domain_busn_res() for root buses, and we should know the bus number range for root buses when we set up the struct pci_host_bridge, so it seems like we should keep the bus number resource there instead of allocating it in this sort of random place.

Then we shouldn't need this weird pci_domain_busn_res_list at all.

Oops, sorry, I totally missed the point here. The point is that for each domain, we get a new 00-ff range of possible bus numbers. This is independent of the host bridges for that domain that may exist. Then each host bridge will allocate a piece of the 00-ff range.

But I do still think get_pci_domain_busn_res() isn't really the best place for this. It seems like it should be at a higher level, connected somehow to domain number allocation, e.g., somewhere related to bridge->domain_nr like the pci_bus_find_domain_nr() path.

Quoted 61 lines
Quoted 61 lines
---
 drivers/pci/probe.c | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)
drivers/pci/probe.c +18 -7
index e19b79821dd6..1327fd820b24 100644
@@ -37,6 +37,7 @@ LIST_HEAD(pci_root_buses);
 EXPORT_SYMBOL(pci_root_buses);
 
 static LIST_HEAD(pci_domain_busn_res_list);
+static DEFINE_MUTEX(pci_domain_busn_res_list_lock);
 
 struct pci_domain_busn_res {
 	struct list_head list;
@@ -47,14 +48,22 @@ struct pci_domain_busn_res {
 static struct resource *get_pci_domain_busn_res(int domain_nr)
 {
 	struct pci_domain_busn_res *r;
+	struct resource *ret;
 
-	list_for_each_entry(r, &pci_domain_busn_res_list, list)
-		if (r->domain_nr == domain_nr)
-			return &r->res;
+	mutex_lock(&pci_domain_busn_res_list_lock);
+
+	list_for_each_entry(r, &pci_domain_busn_res_list, list) {
+		if (r->domain_nr == domain_nr) {
+			ret = &r->res;
+			goto out;
+		}
+	}
 
 	r = kzalloc(sizeof(*r), GFP_KERNEL);
-	if (!r)
-		return NULL;
+	if (!r) {
+		ret = NULL;
+		goto out;
+	}
 
 	r->domain_nr = domain_nr;
 	r->res.start = 0;
@@ -62,8 +71,10 @@ static struct resource *get_pci_domain_busn_res(int domain_nr)
 	r->res.flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED;
 
 	list_add_tail(&r->list, &pci_domain_busn_res_list);
-
-	return &r->res;
+	ret = &r->res;
+out:
+	mutex_unlock(&pci_domain_busn_res_list_lock);
+	return ret;
 }
 
 /*
Signature · 1 line
2.25.1