Skip to contentSkip to conversation

pci: Provide Multiple Error Received support on AER

PATCH Superseded Zhang, Yanmin 2 participants 5 messages Jun 12, 2009 – Jun 15, 2009 PCI

v4 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

PATCH v4 3/3 pci: Provide Multiple Error Received support on AER

When a root port receive the same errors more than once before kernel
process them, the Multiple Error Messages Received flags are set by
hardware. Because root port could only save one kind of correctable error source id and another uncorrectable error source id at the same time, so the second message sender id is lost if the 2 messages are sent from 2 different devices. Below patch searches all devices under the root port when multiple messages are received.

Signed-off-by: Zhang Yanmin <[email protected]>

---

drivers/pci/pcie/aer/aerdrv_core.c +69 -23
diff -Nraup linux-2.6_next_aernoid/drivers/pci/pcie/aer/aerdrv_core.c linux-2.6_next_aermultierror/drivers/pci/pcie/aer/aerdrv_core.c
@@ -145,13 +145,22 @@ static void set_downstream_devices_error
 	pci_walk_bus(dev->subordinate, set_device_error_reporting, &enable);
 }
 
+static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev)
+{
+	if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) {
+		e_info->dev[e_info->error_dev_num ++] = dev;
+		return 1;
+	} else
+		return 0;
+}
+
 static int compare_device_id(struct pci_dev *dev, struct aer_err_info *e_info)
 {
 	if (e_info->id == ((dev->bus->number << 8) | dev->devfn)) {
 		/*
 		 * Device ID match
 		 */
-		e_info->dev = dev;
+		add_error_device(e_info, dev);
 		return 1;
 	}
 
@@ -166,20 +175,38 @@ static int find_device_iter(struct pci_d
 	u32 status;
 	u32 mask;
 	u16 reg16;
+	int result;
 	struct aer_err_info *e_info = (struct aer_err_info *)data;
 
 	/*
 	 * When bus id is equal to 0, it might be a bad id
 	 * reported by root port.
 	 */
-	if (!nosourceid && (PCI_BUS(e_info->id) != 0))
-		return compare_device_id(dev, e_info);
+	if (!nosourceid && (PCI_BUS(e_info->id) != 0)) {
+		result = compare_device_id(dev, e_info);
+		/*
+		 * If there is no multiple error, we stop
+		 * or continue based on the id comparing.
+		 */
+		if (!(e_info->flags & AER_MULTI_ERROR_VALID_FLAG))
+			return result;
+
+		/*
+		 * If there are multiple errors and id does match,
+		 * We need continue to search other devices under
+		 * the root port. Return 0 means that.
+		 */
+		if (result)
+			return 0;
+	}
 
 	/*
-	 * Next is to check when bus id is equal to 0 or
-	 * nosourceid==y. Some ports might lose the bus
-	 * id of error source id. We check AER status
-	 * registers to find the initial reporter.
+	 * When either
+	 * 	1) nosourceid==y;
+	 * 	2) bus id is equal to 0. Some ports might lose the bus
+	 * 		id of error source id;
+	 * 	3) There are multiple errors and prior id comparing fails;
+	 * We check AER status registers to find the initial reporter.
 	 */
 	if (atomic_read(&dev->enable_cnt) == 0)
 		return 0;
@@ -208,8 +235,8 @@ static int find_device_iter(struct pci_d
 				pos + PCI_ERR_COR_MASK,
 				&mask);
 		if (status & ERR_CORRECTABLE_ERROR_MASK & ~mask) {
-			e_info->dev = dev;
-			return 1;
+			add_error_device(e_info, dev);
+			goto added;
 		}
 	} else {
 		pci_read_config_dword(dev,
@@ -219,12 +246,18 @@ static int find_device_iter(struct pci_d
 				pos + PCI_ERR_UNCOR_MASK,
 				&mask);
 		if (status & ERR_UNCORRECTABLE_ERROR_MASK & ~mask) {
-			e_info->dev = dev;
-			return 1;
+			add_error_device(e_info, dev);
+			goto added;
 		}
 	}
 
 	return 0;
+
+added:
+	if (e_info->flags & AER_MULTI_ERROR_VALID_FLAG) {
+		return 0;
+	} else
+		return 1;
 }
 
 /**
@@ -705,6 +738,30 @@ static int get_device_error_info(struct 
 	return AER_SUCCESS;
 }
 
+static inline void aer_process_err_devices(struct pcie_device *p_device,
+			struct aer_err_info *e_info)
+{
+	int i;
+
+	if (e_info->dev[0] == NULL) {
+		printk(KERN_DEBUG "%s->can't find device of ID%04x\n",
+				__func__, e_info->id);
+	}
+
+	for (i = 0; i < e_info->error_dev_num; i ++) {
+		if (e_info->dev[i] == NULL)
+			break;
+
+		if (get_device_error_info(e_info->dev[i], e_info) ==
+				AER_SUCCESS) {
+			aer_print_error(e_info->dev[i], e_info);
+			handle_error_source(p_device,
+					e_info->dev[i],
+					e_info);
+		}
+	}
+}
+
 /**
  * aer_isr_one_error - consume an error detected by root port
  * @p_device: pointer to error root port service device
@@ -747,18 +804,7 @@ static void aer_isr_one_error(struct pci
 			e_info->flags |= AER_MULTI_ERROR_VALID_FLAG;
 
 		find_source_device(p_device->port, e_info);
-		if (e_info->dev == NULL) {
-			printk(KERN_DEBUG "%s->can't find device of ID%04x\n",
-				__func__, e_info->id);
-			continue;
-		}
-		if (get_device_error_info(e_info->dev, e_info) ==
-				AER_SUCCESS) {
-			aer_print_error(e_info->dev, e_info);
-			handle_error_source(p_device,
-				e_info->dev,
-				e_info);
-		}
+		aer_process_err_devices(p_device, e_info);
 	}
 
 	kfree(e_info);
drivers/pci/pcie/aer/aerdrv.h +3 -1
diff -Nraup linux-2.6_next_aernoid/drivers/pci/pcie/aer/aerdrv.h linux-2.6_next_aermultierror/drivers/pci/pcie/aer/aerdrv.h
@@ -57,8 +57,10 @@ struct header_log_regs {
 	unsigned int dw3;
 };
 
+#define AER_MAX_MULTI_ERR_DEVICES	5
 struct aer_err_info {
-	struct pci_dev *dev;
+	struct pci_dev *dev[AER_MAX_MULTI_ERR_DEVICES];
+	int error_dev_num;
 	u16 id;
 	int severity;			/* 0:NONFATAL | 1:FATAL | 2:COR */
 	int flags;
#1 lore raw
Quoted 16 lines · Yanmin

When a root port receive the same errors more than once before kernel
process them, the Multiple Error Messages Received flags are set by
hardware. Because root port could only save one kind of correctable error source id and another uncorrectable error source id at the same time, so the second message sender id is lost if the 2 messages are sent from 2 different devices. Below patch searches all devices under the root port when multiple messages are received.

Signed-off-by: Zhang Yanmin <[email protected]>

---

diff -Nraup linux-2.6_next_aernoid/drivers/pci/pcie/aer/aerdrv_core.c linux-2.6_next_aermultierror/drivers/pci/pcie/aer/aerdrv_core.c

drivers/pci/pcie/aer/aerdrv_core.c static void set_downstream_devices_error · line 151

Quoted 7 lines
drivers/pci/pcie/aer/aerdrv_core.c +4 -0
@@ -145,13 +145,22 @@ static void set_downstream_devices_error
 	pci_walk_bus(dev->subordinate, set_device_error_reporting, &enable);
 }
 
+static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev)
+{
+	if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) {
+		e_info->dev[e_info->error_dev_num ++] = dev;

checkpatch reports:
ERROR: space prohibited before that '++' (ctx:WxB) #46: FILE: drivers/pci/pcie/aer/aerdrv_core.c:151:
+ e_info->dev[e_info->error_dev_num ++] = dev;

Personally I would prefer:

		e_info->dev[e_info->error_dev_num] = dev;
		e_info->error_dev_num++;

drivers/pci/pcie/aer/aerdrv_core.c static void set_downstream_devices_error · line 156

Quoted 5 lines
drivers/pci/pcie/aer/aerdrv_core.c +5 -0
+		return 1;
+	} else
+		return 0;
+}
+

This function is now doing more than just comparing device ID's. Perhaps you could rename it or put call add_error_device after compare_device_id in find_device_iter?

Quoted 75 lines
drivers/pci/pcie/aer/aerdrv_core.c +29 -11
 static int compare_device_id(struct pci_dev *dev, struct aer_err_info *e_info)
 {
 	if (e_info->id == ((dev->bus->number << 8) | dev->devfn)) {
 		/*
 		 * Device ID match
 		 */
-		e_info->dev = dev;
+		add_error_device(e_info, dev);
 		return 1;
 	}
 
@@ -166,20 +175,38 @@ static int find_device_iter(struct pci_d
 	u32 status;
 	u32 mask;
 	u16 reg16;
+	int result;
 	struct aer_err_info *e_info = (struct aer_err_info *)data;
 
 	/*
 	 * When bus id is equal to 0, it might be a bad id
 	 * reported by root port.
 	 */
-	if (!nosourceid && (PCI_BUS(e_info->id) != 0))
-		return compare_device_id(dev, e_info);
+	if (!nosourceid && (PCI_BUS(e_info->id) != 0)) {
+		result = compare_device_id(dev, e_info);
+		/*
+		 * If there is no multiple error, we stop
+		 * or continue based on the id comparing.
+		 */
+		if (!(e_info->flags & AER_MULTI_ERROR_VALID_FLAG))
+			return result;
+
+		/*
+		 * If there are multiple errors and id does match,
+		 * We need continue to search other devices under
+		 * the root port. Return 0 means that.
+		 */
+		if (result)
+			return 0;
+	}
 
 	/*
-	 * Next is to check when bus id is equal to 0 or
-	 * nosourceid==y. Some ports might lose the bus
-	 * id of error source id. We check AER status
-	 * registers to find the initial reporter.
+	 * When either
+	 * 	1) nosourceid==y;
+	 * 	2) bus id is equal to 0. Some ports might lose the bus
+	 * 		id of error source id;
+	 * 	3) There are multiple errors and prior id comparing fails;
+	 * We check AER status registers to find the initial reporter.
 	 */
 	if (atomic_read(&dev->enable_cnt) == 0)
 		return 0;
@@ -208,8 +235,8 @@ static int find_device_iter(struct pci_d
 				pos + PCI_ERR_COR_MASK,
 				&mask);
 		if (status & ERR_CORRECTABLE_ERROR_MASK & ~mask) {
-			e_info->dev = dev;
-			return 1;
+			add_error_device(e_info, dev);
+			goto added;
 		}
 	} else {
 		pci_read_config_dword(dev,
@@ -219,12 +246,18 @@ static int find_device_iter(struct pci_d
 				pos + PCI_ERR_UNCOR_MASK,
 				&mask);
 		if (status & ERR_UNCORRECTABLE_ERROR_MASK & ~mask) {
-			e_info->dev = dev;
-			return 1;
+			add_error_device(e_info, dev);
+			goto added;

drivers/pci/pcie/aer/aerdrv_core.c static int find_device_iter(struct pci_d · line 260

Quoted 10 lines
drivers/pci/pcie/aer/aerdrv_core.c +6 -0
 		}
 	}
 
 	return 0;
+
+added:
+	if (e_info->flags & AER_MULTI_ERROR_VALID_FLAG) {
+		return 0;
+	} else
+		return 1;

checkpatch reports:
WARNING: braces {} are not necessary for any arm of this statement #133: FILE: drivers/pci/pcie/aer/aerdrv_core.c:257:
+ if (e_info->flags & AER_MULTI_ERROR_VALID_FLAG) {
[...]
+ } else
[...]

drivers/pci/pcie/aer/aerdrv_core.c static int get_device_error_info(struct · line 746

Quoted 13 lines
drivers/pci/pcie/aer/aerdrv_core.c +6 -0
 }
 
 /**
@@ -705,6 +738,30 @@ static int get_device_error_info(struct 
 	return AER_SUCCESS;
 }
 
+static inline void aer_process_err_devices(struct pcie_device *p_device,
+			struct aer_err_info *e_info)
+{
+	int i;
+
+	if (e_info->dev[0] == NULL) {

Minor not. Can we use

          if (!e_info->dev[0]) {

drivers/pci/pcie/aer/aerdrv_core.c static int get_device_error_info(struct · line 748

Quoted 2 lines
drivers/pci/pcie/aer/aerdrv_core.c +2 -0
+		printk(KERN_DEBUG "%s->can't find device of ID%04x\n",
+				__func__, e_info->id);

I suspect we don't want to embed the function name here, and use dev_printk.

drivers/pci/pcie/aer/aerdrv_core.c static int get_device_error_info(struct · line 751

Quoted 3 lines
drivers/pci/pcie/aer/aerdrv_core.c +3 -0
+	}
+
+	for (i = 0; i < e_info->error_dev_num; i ++) {

checkpatch reports:
ERROR: space prohibited before that '++' (ctx:WxB) #154: FILE: drivers/pci/pcie/aer/aerdrv_core.c:751:
+ for (i = 0; i < e_info->error_dev_num; i +

drivers/pci/pcie/aer/aerdrv_core.c static int get_device_error_info(struct · line 752

Quoted 1 line
drivers/pci/pcie/aer/aerdrv_core.c +1 -0
+		if (e_info->dev[i] == NULL)

again if (!e_info->dev[i])

You could also put this check in the for loop.

Quoted 39 lines
drivers/pci/pcie/aer/aerdrv_core.c +13 -12
+			break;
+
+		if (get_device_error_info(e_info->dev[i], e_info) ==
+				AER_SUCCESS) {
+			aer_print_error(e_info->dev[i], e_info);
+			handle_error_source(p_device,
+					e_info->dev[i],
+					e_info);
+		}
+	}
+}
+
 /**
  * aer_isr_one_error - consume an error detected by root port
  * @p_device: pointer to error root port service device
@@ -747,18 +804,7 @@ static void aer_isr_one_error(struct pci
 			e_info->flags |= AER_MULTI_ERROR_VALID_FLAG;
 
 		find_source_device(p_device->port, e_info);
-		if (e_info->dev == NULL) {
-			printk(KERN_DEBUG "%s->can't find device of ID%04x\n",
-				__func__, e_info->id);
-			continue;
-		}
-		if (get_device_error_info(e_info->dev, e_info) ==
-				AER_SUCCESS) {
-			aer_print_error(e_info->dev, e_info);
-			handle_error_source(p_device,
-				e_info->dev,
-				e_info);
-		}
+		aer_process_err_devices(p_device, e_info);
 	}
 
 	kfree(e_info);

diff -Nraup linux-2.6_next_aernoid/drivers/pci/pcie/aer/aerdrv.h linux-2.6_next_aermultierror/drivers/pci/pcie/aer/aerdrv.h

drivers/pci/pcie/aer/aerdrv.h struct header_log_regs { · line 60

Quoted 4 lines
drivers/pci/pcie/aer/aerdrv.h +1 -0
@@ -57,8 +57,10 @@ struct header_log_regs {
 	unsigned int dw3;
 };
 
+#define AER_MAX_MULTI_ERR_DEVICES	5

Is this number arbitrary or in the spec somewhere?

Quoted 14 lines
drivers/pci/pcie/aer/aerdrv.h +2 -2
 struct aer_err_info {
-	struct pci_dev *dev;
+	struct pci_dev *dev[AER_MAX_MULTI_ERR_DEVICES];
+	int error_dev_num;
 	u16 id;
 	int severity;			/* 0:NONFATAL | 1:FATAL | 2:COR */
 	int flags;
 
 
--

To unsubscribe from this list: send the line "unsubscribe linux-pci" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Signature · 2 lines
Andrew Patterson
Hewlett-Packard
#2 lore raw
Quoted 28 lines · Andrew Patterson
Quoted 16 lines · Yanmin

When a root port receive the same errors more than once before kernel
process them, the Multiple Error Messages Received flags are set by
hardware. Because root port could only save one kind of correctable error source id and another uncorrectable error source id at the same time, so the second message sender id is lost if the 2 messages are sent from 2 different devices. Below patch searches all devices under the root port when multiple messages are received.

Signed-off-by: Zhang Yanmin <[email protected]>

---

diff -Nraup linux-2.6_next_aernoid/drivers/pci/pcie/aer/aerdrv_core.c linux-2.6_next_aermultierror/drivers/pci/pcie/aer/aerdrv_core.c

drivers/pci/pcie/aer/aerdrv_core.c static void set_downstream_devices_error · line 151

Quoted 7 lines
drivers/pci/pcie/aer/aerdrv_core.c +4 -0
@@ -145,13 +145,22 @@ static void set_downstream_devices_error
 	pci_walk_bus(dev->subordinate, set_device_error_reporting, &enable);
 }
 
+static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev)
+{
+	if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) {
+		e_info->dev[e_info->error_dev_num ++] = dev;

checkpatch reports:
ERROR: space prohibited before that '++' (ctx:WxB) #46: FILE: drivers/pci/pcie/aer/aerdrv_core.c:151:

drivers/pci/pcie/aer/aerdrv_core.c static void set_downstream_devices_error · line 151

Quoted 1 line
drivers/pci/pcie/aer/aerdrv_core.c +1 -0
+		e_info->dev[e_info->error_dev_num ++] = dev;

Thanks. I will change it and use checkpatch to check it.

Quoted 5 lines

Personally I would prefer:

		e_info->dev[e_info->error_dev_num] = dev;
		e_info->error_dev_num++;
Quoted 10 lines

drivers/pci/pcie/aer/aerdrv_core.c static void set_downstream_devices_error · line 156

Quoted 5 lines
drivers/pci/pcie/aer/aerdrv_core.c +5 -0
+		return 1;
+	} else
+		return 0;
+}
+

This function is now doing more than just comparing device ID's. Perhaps you could rename it or put call add_error_device after compare_device_id in find_device_iter?

I will move the call add_error_device to find_device_iter.

Quoted 78 lines
Quoted 78 lines
drivers/pci/pcie/aer/aerdrv_core.c +29 -11
 static int compare_device_id(struct pci_dev *dev, struct aer_err_info *e_info)
 {
 	if (e_info->id == ((dev->bus->number << 8) | dev->devfn)) {
 		/*
 		 * Device ID match
 		 */
-		e_info->dev = dev;
+		add_error_device(e_info, dev);
 		return 1;
 	}
 
@@ -166,20 +175,38 @@ static int find_device_iter(struct pci_d
 	u32 status;
 	u32 mask;
 	u16 reg16;
+	int result;
 	struct aer_err_info *e_info = (struct aer_err_info *)data;
 
 	/*
 	 * When bus id is equal to 0, it might be a bad id
 	 * reported by root port.
 	 */
-	if (!nosourceid && (PCI_BUS(e_info->id) != 0))
-		return compare_device_id(dev, e_info);
+	if (!nosourceid && (PCI_BUS(e_info->id) != 0)) {
+		result = compare_device_id(dev, e_info);
+		/*
+		 * If there is no multiple error, we stop
+		 * or continue based on the id comparing.
+		 */
+		if (!(e_info->flags & AER_MULTI_ERROR_VALID_FLAG))
+			return result;
+
+		/*
+		 * If there are multiple errors and id does match,
+		 * We need continue to search other devices under
+		 * the root port. Return 0 means that.
+		 */
+		if (result)
+			return 0;
+	}
 
 	/*
-	 * Next is to check when bus id is equal to 0 or
-	 * nosourceid==y. Some ports might lose the bus
-	 * id of error source id. We check AER status
-	 * registers to find the initial reporter.
+	 * When either
+	 * 	1) nosourceid==y;
+	 * 	2) bus id is equal to 0. Some ports might lose the bus
+	 * 		id of error source id;
+	 * 	3) There are multiple errors and prior id comparing fails;
+	 * We check AER status registers to find the initial reporter.
 	 */
 	if (atomic_read(&dev->enable_cnt) == 0)
 		return 0;
@@ -208,8 +235,8 @@ static int find_device_iter(struct pci_d
 				pos + PCI_ERR_COR_MASK,
 				&mask);
 		if (status & ERR_CORRECTABLE_ERROR_MASK & ~mask) {
-			e_info->dev = dev;
-			return 1;
+			add_error_device(e_info, dev);
+			goto added;
 		}
 	} else {
 		pci_read_config_dword(dev,
@@ -219,12 +246,18 @@ static int find_device_iter(struct pci_d
 				pos + PCI_ERR_UNCOR_MASK,
 				&mask);
 		if (status & ERR_UNCORRECTABLE_ERROR_MASK & ~mask) {
-			e_info->dev = dev;
-			return 1;
+			add_error_device(e_info, dev);
+			goto added;
 		}
 	}
 
Quoted 11 lines

drivers/pci/pcie/aer/aerdrv_core.c static int find_device_iter(struct pci_d · line 260

Quoted 7 lines
drivers/pci/pcie/aer/aerdrv_core.c +6 -0
 	return 0;
+
+added:
+	if (e_info->flags & AER_MULTI_ERROR_VALID_FLAG) {
+		return 0;
+	} else
+		return 1;

checkpatch reports:
WARNING: braces {} are not necessary for any arm of this statement #133: FILE: drivers/pci/pcie/aer/aerdrv_core.c:257:

I will change it.

Quoted 13 lines
drivers/pci/pcie/aer/aerdrv_core.c +2 -0
+	if (e_info->flags & AER_MULTI_ERROR_VALID_FLAG) {
[...]
+	} else
[...]
Quoted 9 lines
drivers/pci/pcie/aer/aerdrv_core.c +0 -0
 }
 
 /**
@@ -705,6 +738,30 @@ static int get_device_error_info(struct 
 	return AER_SUCCESS;
 }
 
Quoted 7 lines

drivers/pci/pcie/aer/aerdrv_core.c static int get_device_error_info(struct · line 746

Quoted 6 lines
drivers/pci/pcie/aer/aerdrv_core.c +6 -0
+static inline void aer_process_err_devices(struct pcie_device *p_device,
+			struct aer_err_info *e_info)
+{
+	int i;
+
+	if (e_info->dev[0] == NULL) {

Minor not. Can we use

Yes. We can.

Quoted 7 lines
          if (!e_info->dev[0]) {

drivers/pci/pcie/aer/aerdrv_core.c static int get_device_error_info(struct · line 748

Quoted 2 lines
drivers/pci/pcie/aer/aerdrv_core.c +2 -0
+		printk(KERN_DEBUG "%s->can't find device of ID%04x\n",
+				__func__, e_info->id);

I suspect we don't want to embed the function name here, and use dev_printk.

Ok.

Review

Quoted 8 lines

drivers/pci/pcie/aer/aerdrv_core.c static int get_device_error_info(struct · line 751

Quoted 4 lines
drivers/pci/pcie/aer/aerdrv_core.c +3 -0
+	}
+
+	for (i = 0; i < e_info->error_dev_num; i ++) {

checkpatch reports:
ERROR: space prohibited before that '++' (ctx:WxB) #154: FILE: drivers/pci/pcie/aer/aerdrv_core.c:751:

(quoted diff) +1 -0
+	for (i = 0; i < e_info->error_dev_num; i +

I will change it.

Quoted 4 lines

drivers/pci/pcie/aer/aerdrv_core.c static int get_device_error_info(struct · line 752

Quoted 3 lines
drivers/pci/pcie/aer/aerdrv_core.c +1 -0
+		if (e_info->dev[i] == NULL)

again if (!e_info->dev[i])

Will do.

Quoted 2 lines

You could also put this check in the for loop.

I planed to, but one guy helped me test it within a guest OS on XEN and reported a weired oops of guest OS. She said useing e_info->error_dev_num could avoid the oops.

Quoted 44 lines
Quoted 44 lines
drivers/pci/pcie/aer/aerdrv_core.c +13 -12
+			break;
+
+		if (get_device_error_info(e_info->dev[i], e_info) ==
+				AER_SUCCESS) {
+			aer_print_error(e_info->dev[i], e_info);
+			handle_error_source(p_device,
+					e_info->dev[i],
+					e_info);
+		}
+	}
+}
+
 /**
  * aer_isr_one_error - consume an error detected by root port
  * @p_device: pointer to error root port service device
@@ -747,18 +804,7 @@ static void aer_isr_one_error(struct pci
 			e_info->flags |= AER_MULTI_ERROR_VALID_FLAG;
 
 		find_source_device(p_device->port, e_info);
-		if (e_info->dev == NULL) {
-			printk(KERN_DEBUG "%s->can't find device of ID%04x\n",
-				__func__, e_info->id);
-			continue;
-		}
-		if (get_device_error_info(e_info->dev, e_info) ==
-				AER_SUCCESS) {
-			aer_print_error(e_info->dev, e_info);
-			handle_error_source(p_device,
-				e_info->dev,
-				e_info);
-		}
+		aer_process_err_devices(p_device, e_info);
 	}
 
 	kfree(e_info);

diff -Nraup linux-2.6_next_aernoid/drivers/pci/pcie/aer/aerdrv.h linux-2.6_next_aermultierror/drivers/pci/pcie/aer/aerdrv.h

drivers/pci/pcie/aer/aerdrv.h +0 -0
@@ -57,8 +57,10 @@ struct header_log_regs {
 	unsigned int dw3;
 };
 
Quoted 2 lines

drivers/pci/pcie/aer/aerdrv.h struct header_log_regs { · line 60

Quoted 1 line
drivers/pci/pcie/aer/aerdrv.h +1 -0
+#define AER_MAX_MULTI_ERR_DEVICES	5

Is this number arbitrary or in the spec somewhere?

It's arbitrary and not spec. The startpoint is it's very rare that there are more than 5 devices under the same root port reporting errors at the same time. It's hard to say number 5 is the best. I just don't want the array is big.

Quoted 10 lines
Quoted 10 lines
drivers/pci/pcie/aer/aerdrv.h +2 -1
 struct aer_err_info {
-	struct pci_dev *dev;
+	struct pci_dev *dev[AER_MAX_MULTI_ERR_DEVICES];
+	int error_dev_num;
 	u16 id;
 	int severity;			/* 0:NONFATAL | 1:FATAL | 2:COR */
 	int flags;
#3 lore raw
Quoted 185 lines · Yanmin
Quoted 28 lines · Andrew Patterson
Quoted 16 lines · Yanmin

When a root port receive the same errors more than once before kernel
process them, the Multiple Error Messages Received flags are set by
hardware. Because root port could only save one kind of correctable error source id and another uncorrectable error source id at the same time, so the second message sender id is lost if the 2 messages are sent from 2 different devices. Below patch searches all devices under the root port when multiple messages are received.

Signed-off-by: Zhang Yanmin <[email protected]>

---

diff -Nraup linux-2.6_next_aernoid/drivers/pci/pcie/aer/aerdrv_core.c linux-2.6_next_aermultierror/drivers/pci/pcie/aer/aerdrv_core.c

drivers/pci/pcie/aer/aerdrv_core.c static void set_downstream_devices_error · line 151

Quoted 7 lines
drivers/pci/pcie/aer/aerdrv_core.c +4 -0
@@ -145,13 +145,22 @@ static void set_downstream_devices_error
 	pci_walk_bus(dev->subordinate, set_device_error_reporting, &enable);
 }
 
+static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev)
+{
+	if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) {
+		e_info->dev[e_info->error_dev_num ++] = dev;

checkpatch reports:
ERROR: space prohibited before that '++' (ctx:WxB) #46: FILE: drivers/pci/pcie/aer/aerdrv_core.c:151:

drivers/pci/pcie/aer/aerdrv_core.c static void set_downstream_devices_error · line 151

Quoted 1 line
drivers/pci/pcie/aer/aerdrv_core.c +1 -0
+		e_info->dev[e_info->error_dev_num ++] = dev;

Thanks. I will change it and use checkpatch to check it.

Quoted 5 lines

Personally I would prefer:

		e_info->dev[e_info->error_dev_num] = dev;
		e_info->error_dev_num++;
Quoted 10 lines

drivers/pci/pcie/aer/aerdrv_core.c static void set_downstream_devices_error · line 156

Quoted 5 lines
drivers/pci/pcie/aer/aerdrv_core.c +5 -0
+		return 1;
+	} else
+		return 0;
+}
+

This function is now doing more than just comparing device ID's. Perhaps you could rename it or put call add_error_device after compare_device_id in find_device_iter?

I will move the call add_error_device to find_device_iter.

Quoted 78 lines
Quoted 78 lines
drivers/pci/pcie/aer/aerdrv_core.c +29 -11
 static int compare_device_id(struct pci_dev *dev, struct aer_err_info *e_info)
 {
 	if (e_info->id == ((dev->bus->number << 8) | dev->devfn)) {
 		/*
 		 * Device ID match
 		 */
-		e_info->dev = dev;
+		add_error_device(e_info, dev);
 		return 1;
 	}
 
@@ -166,20 +175,38 @@ static int find_device_iter(struct pci_d
 	u32 status;
 	u32 mask;
 	u16 reg16;
+	int result;
 	struct aer_err_info *e_info = (struct aer_err_info *)data;
 
 	/*
 	 * When bus id is equal to 0, it might be a bad id
 	 * reported by root port.
 	 */
-	if (!nosourceid && (PCI_BUS(e_info->id) != 0))
-		return compare_device_id(dev, e_info);
+	if (!nosourceid && (PCI_BUS(e_info->id) != 0)) {
+		result = compare_device_id(dev, e_info);
+		/*
+		 * If there is no multiple error, we stop
+		 * or continue based on the id comparing.
+		 */
+		if (!(e_info->flags & AER_MULTI_ERROR_VALID_FLAG))
+			return result;
+
+		/*
+		 * If there are multiple errors and id does match,
+		 * We need continue to search other devices under
+		 * the root port. Return 0 means that.
+		 */
+		if (result)
+			return 0;
+	}
 
 	/*
-	 * Next is to check when bus id is equal to 0 or
-	 * nosourceid==y. Some ports might lose the bus
-	 * id of error source id. We check AER status
-	 * registers to find the initial reporter.
+	 * When either
+	 * 	1) nosourceid==y;
+	 * 	2) bus id is equal to 0. Some ports might lose the bus
+	 * 		id of error source id;
+	 * 	3) There are multiple errors and prior id comparing fails;
+	 * We check AER status registers to find the initial reporter.
 	 */
 	if (atomic_read(&dev->enable_cnt) == 0)
 		return 0;
@@ -208,8 +235,8 @@ static int find_device_iter(struct pci_d
 				pos + PCI_ERR_COR_MASK,
 				&mask);
 		if (status & ERR_CORRECTABLE_ERROR_MASK & ~mask) {
-			e_info->dev = dev;
-			return 1;
+			add_error_device(e_info, dev);
+			goto added;
 		}
 	} else {
 		pci_read_config_dword(dev,
@@ -219,12 +246,18 @@ static int find_device_iter(struct pci_d
 				pos + PCI_ERR_UNCOR_MASK,
 				&mask);
 		if (status & ERR_UNCORRECTABLE_ERROR_MASK & ~mask) {
-			e_info->dev = dev;
-			return 1;
+			add_error_device(e_info, dev);
+			goto added;
 		}
 	}
 
Quoted 11 lines

drivers/pci/pcie/aer/aerdrv_core.c static int find_device_iter(struct pci_d · line 260

Quoted 7 lines
drivers/pci/pcie/aer/aerdrv_core.c +6 -0
 	return 0;
+
+added:
+	if (e_info->flags & AER_MULTI_ERROR_VALID_FLAG) {
+		return 0;
+	} else
+		return 1;

checkpatch reports:
WARNING: braces {} are not necessary for any arm of this statement #133: FILE: drivers/pci/pcie/aer/aerdrv_core.c:257:

I will change it.

Quoted 13 lines
drivers/pci/pcie/aer/aerdrv_core.c +2 -0
+	if (e_info->flags & AER_MULTI_ERROR_VALID_FLAG) {
[...]
+	} else
[...]
Quoted 9 lines
drivers/pci/pcie/aer/aerdrv_core.c +0 -0
 }
 
 /**
@@ -705,6 +738,30 @@ static int get_device_error_info(struct 
 	return AER_SUCCESS;
 }
 
Quoted 7 lines

drivers/pci/pcie/aer/aerdrv_core.c static int get_device_error_info(struct · line 746

Quoted 6 lines
drivers/pci/pcie/aer/aerdrv_core.c +6 -0
+static inline void aer_process_err_devices(struct pcie_device *p_device,
+			struct aer_err_info *e_info)
+{
+	int i;
+
+	if (e_info->dev[0] == NULL) {

Minor not. Can we use

Yes. We can.

Quoted 7 lines
          if (!e_info->dev[0]) {

drivers/pci/pcie/aer/aerdrv_core.c static int get_device_error_info(struct · line 748

Quoted 2 lines
drivers/pci/pcie/aer/aerdrv_core.c +2 -0
+		printk(KERN_DEBUG "%s->can't find device of ID%04x\n",
+				__func__, e_info->id);

I suspect we don't want to embed the function name here, and use dev_printk.

Ok.

Review

Quoted 8 lines

drivers/pci/pcie/aer/aerdrv_core.c static int get_device_error_info(struct · line 751

Quoted 4 lines
drivers/pci/pcie/aer/aerdrv_core.c +3 -0
+	}
+
+	for (i = 0; i < e_info->error_dev_num; i ++) {

checkpatch reports:
ERROR: space prohibited before that '++' (ctx:WxB) #154: FILE: drivers/pci/pcie/aer/aerdrv_core.c:751:

(quoted diff) +1 -0
+	for (i = 0; i < e_info->error_dev_num; i +

I will change it.

Quoted 4 lines

drivers/pci/pcie/aer/aerdrv_core.c static int get_device_error_info(struct · line 752

Quoted 3 lines
drivers/pci/pcie/aer/aerdrv_core.c +1 -0
+		if (e_info->dev[i] == NULL)

again if (!e_info->dev[i])

Quoted 7 lines

Will do.

Quoted 2 lines

You could also put this check in the for loop.

I planed to, but one guy helped me test it within a guest OS on XEN and reported a weired oops of guest OS. She said useing e_info->error_dev_num could avoid the oops.

I think something like:

  for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++)

is functionally equivalent.

Quoted 45 lines
Quoted 45 lines
Quoted 45 lines
drivers/pci/pcie/aer/aerdrv_core.c +13 -12
+			break;
+
+		if (get_device_error_info(e_info->dev[i], e_info) ==
+				AER_SUCCESS) {
+			aer_print_error(e_info->dev[i], e_info);
+			handle_error_source(p_device,
+					e_info->dev[i],
+					e_info);
+		}
+	}
+}
+
 /**
  * aer_isr_one_error - consume an error detected by root port
  * @p_device: pointer to error root port service device
@@ -747,18 +804,7 @@ static void aer_isr_one_error(struct pci
 			e_info->flags |= AER_MULTI_ERROR_VALID_FLAG;
 
 		find_source_device(p_device->port, e_info);
-		if (e_info->dev == NULL) {
-			printk(KERN_DEBUG "%s->can't find device of ID%04x\n",
-				__func__, e_info->id);
-			continue;
-		}
-		if (get_device_error_info(e_info->dev, e_info) ==
-				AER_SUCCESS) {
-			aer_print_error(e_info->dev, e_info);
-			handle_error_source(p_device,
-				e_info->dev,
-				e_info);
-		}
+		aer_process_err_devices(p_device, e_info);
 	}
 
 	kfree(e_info);

diff -Nraup linux-2.6_next_aernoid/drivers/pci/pcie/aer/aerdrv.h linux-2.6_next_aermultierror/drivers/pci/pcie/aer/aerdrv.h

drivers/pci/pcie/aer/aerdrv.h +0 -0
@@ -57,8 +57,10 @@ struct header_log_regs {
 	unsigned int dw3;
 };
 
Quoted 3 lines
Quoted 2 lines

drivers/pci/pcie/aer/aerdrv.h struct header_log_regs { · line 60

Quoted 1 line
drivers/pci/pcie/aer/aerdrv.h +1 -0
+#define AER_MAX_MULTI_ERR_DEVICES	5

Is this number arbitrary or in the spec somewhere?

It's arbitrary and not spec.

I suspected so.

Quoted 2 lines

The startpoint is it's very rare that there are more
than 5 devices under the same root port reporting errors at the same time.

Agreed.

Quoted 2 lines

It's hard
to say number 5 is the best. I just don't want the array is big.

I don't have a problem with that decision. But you might add a comment saying so, e.g.,

#define AER_MAX_MULTI_ERR_DEVICES 5 /* Not likely to have more */

Quoted 13 lines
Quoted 11 lines
Quoted 11 lines
drivers/pci/pcie/aer/aerdrv.h +2 -1
 struct aer_err_info {
-	struct pci_dev *dev;
+	struct pci_dev *dev[AER_MAX_MULTI_ERR_DEVICES];
+	int error_dev_num;
 	u16 id;
 	int severity;			/* 0:NONFATAL | 1:FATAL | 2:COR */
 	int flags;
Signature · 2 lines
Andrew Patterson
Hewlett-Packard
#4 lore raw
Quoted 32 lines · Andrew Patterson
Quoted 23 lines · Yanmin
Quoted 15 lines · Andrew Patterson

drivers/pci/pcie/aer/aerdrv_core.c static int get_device_error_info(struct · line 751

Quoted 11 lines · Yanmin

When a root port receive the same errors more than once before kernel
process them, the Multiple Error Messages Received flags are set by
hardware. Because root port could only save one kind of correctable error source id and another uncorrectable error source id at the same time, so the second message sender id is lost if the 2 messages are sent from 2 different devices. Below patch searches all devices under the root port when multiple messages are received.

Signed-off-by: Zhang Yanmin <[email protected]>

drivers/pci/pcie/aer/aerdrv_core.c +1 -0
+	for (i = 0; i < e_info->error_dev_num; i ++) {

checkpatch reports:
ERROR: space prohibited before that '++' (ctx:WxB) #154: FILE: drivers/pci/pcie/aer/aerdrv_core.c:751:

Review

Quoted 1 line
(quoted diff) +1 -0
+	for (i = 0; i < e_info->error_dev_num; i +

I will change it.

Quoted 4 lines

drivers/pci/pcie/aer/aerdrv_core.c static int get_device_error_info(struct · line 752

Quoted 3 lines
drivers/pci/pcie/aer/aerdrv_core.c +1 -0
+		if (e_info->dev[i] == NULL)

again if (!e_info->dev[i])

Quoted 7 lines

Will do.

Quoted 2 lines

You could also put this check in the for loop.

I planed to, but one guy helped me test it within a guest OS on XEN and reported a weired oops of guest OS. She said useing e_info->error_dev_num could avoid the oops.

Quoted 5 lines

I think something like:

  for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++)

is functionally equivalent.

I changed it.

Quoted 57 lines
Quoted 46 lines
Quoted 46 lines
Quoted 46 lines
drivers/pci/pcie/aer/aerdrv_core.c +13 -12
+			break;
+
+		if (get_device_error_info(e_info->dev[i], e_info) ==
+				AER_SUCCESS) {
+			aer_print_error(e_info->dev[i], e_info);
+			handle_error_source(p_device,
+					e_info->dev[i],
+					e_info);
+		}
+	}
+}
+
 /**
  * aer_isr_one_error - consume an error detected by root port
  * @p_device: pointer to error root port service device
@@ -747,18 +804,7 @@ static void aer_isr_one_error(struct pci
 			e_info->flags |= AER_MULTI_ERROR_VALID_FLAG;
 
 		find_source_device(p_device->port, e_info);
-		if (e_info->dev == NULL) {
-			printk(KERN_DEBUG "%s->can't find device of ID%04x\n",
-				__func__, e_info->id);
-			continue;
-		}
-		if (get_device_error_info(e_info->dev, e_info) ==
-				AER_SUCCESS) {
-			aer_print_error(e_info->dev, e_info);
-			handle_error_source(p_device,
-				e_info->dev,
-				e_info);
-		}
+		aer_process_err_devices(p_device, e_info);
 	}
 
 	kfree(e_info);

diff -Nraup linux-2.6_next_aernoid/drivers/pci/pcie/aer/aerdrv.h linux-2.6_next_aermultierror/drivers/pci/pcie/aer/aerdrv.h

drivers/pci/pcie/aer/aerdrv.h +0 -0
@@ -57,8 +57,10 @@ struct header_log_regs {
 	unsigned int dw3;
 };
 
Quoted 3 lines
Quoted 2 lines

drivers/pci/pcie/aer/aerdrv.h struct header_log_regs { · line 60

Quoted 1 line
drivers/pci/pcie/aer/aerdrv.h +1 -0
+#define AER_MAX_MULTI_ERR_DEVICES	5

Is this number arbitrary or in the spec somewhere?

It's arbitrary and not spec.

I suspected so.

Quoted 2 lines

The startpoint is it's very rare that there are more
than 5 devices under the same root port reporting errors at the same time.

Agreed.

Quoted 7 lines
Quoted 2 lines

It's hard
to say number 5 is the best. I just don't want the array is big.

I don't have a problem with that decision. But you might add a comment saying so, e.g.,

#define AER_MAX_MULTI_ERR_DEVICES 5 /* Not likely to have more */

Added.

Thanks Andrew.