pci: pcie: AER: Fix logging of Correctable errors
Subsystems (2)
PATCH 1/1 pci: pcie: AER: Fix logging of Correctable errors
The AER documentation indicates that correctable (severity=Corrected) errors should be output as a warning so that users can filter these errors if they choose to; This functionality does not appear to have been implemented.
This patch modifies the functions aer_print_error and __aer_print_error
to send correctable errors as a warning (pci_warn), rather than as an error (pci_err). It partially addresses several bugs in relation to kernel message buffer
spam for misbehaving devices - the root cause (possibly device firmware?) isn't addressed, but the dmesg output is less alarming for end users, and can
be filtered separately from uncorrectable errors. This should hopefully reduce the need for users to disable AER to suppress corrected errors.
--- drivers/pci/pcie/aer.c | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-)
drivers/pci/pcie/aer.c +26 -10
@@ -662,12 +662,18 @@ static void __aer_print_error(struct pci_dev *dev, errmsg = i <ARRAY_SIZE (aer_uncorrectable_error_string) ? aer_uncorrectable_error_string[i] :NULL ; -if (errmsg) -pci_err (dev," [%2d] %-22s%s\n" , i, errmsg, - info->first_error == i ?" (First)" :"" ); -else +if (errmsg) { +if (info->severity ==AER_CORRECTABLE ) { +pci_warn (dev," [%2d] %-22s%s\n" , i, errmsg, + info->first_error == i ?" (First)" :"" ); + }else { +pci_err (dev," [%2d] %-22s%s\n" , i, errmsg, + info->first_error == i ?" (First)" :"" ); + } + }else {pci_err (dev," [%2d] Unknown Error Bit%s\n" , i, info->first_error == i ?" (First)" :"" ); + } }pci_dev_aer_stats_incr (dev, info); }
@@ -686,13 +692,23 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) layer =AER_GET_LAYER_ERROR (info->severity, info->status); agent =AER_GET_AGENT (info->severity, info->status); -pci_err (dev,"PCIe Bus Error: severity=%s, type=%s, (%s)\n" , - aer_error_severity_string[info->severity], - aer_error_layer[layer], aer_agent_string[agent]); +if (info->severity ==AER_CORRECTABLE ) { +pci_warn (dev,"PCIe Bus Error: severity=%s, type=%s, (%s)\n" , + aer_error_severity_string[info->severity], + aer_error_layer[layer], aer_agent_string[agent]); -pci_err (dev," device [%04x:%04x] error status/mask=%08x/%08x\n" , - dev->vendor, dev->device, - info->status, info->mask); +pci_warn (dev," device [%04x:%04x] error status/mask=%08x/%08x\n" , + dev->vendor, dev->device, + info->status, info->mask); + }else { +pci_err (dev,"PCIe Bus Error: severity=%s, type=%s, (%s)\n" , + aer_error_severity_string[info->severity], + aer_error_layer[layer], aer_agent_string[agent]); + +pci_err (dev," device [%04x:%04x] error status/mask=%08x/%08x\n" , + dev->vendor, dev->device, + info->status, info->mask); + }__aer_print_error (dev, info);
Signature · 1 line
2.26.2
Quoted 4 lines · Matt Jolly
drivers/pci/pcie/aer.c +4
-0+pci_warn (dev," device [%04x:%04x] error status/mask=%08x/%08x\n" , + dev->vendor, dev->device, + info->status, info->mask); + }else {
<snip>
drivers/pci/pcie/aer.c void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) · line 710
Quoted 3 lines
drivers/pci/pcie/aer.c +3
-0+pci_err (dev," device [%04x:%04x] error status/mask=%08x/%08x\n" , + dev->vendor, dev->device, + info->status, info->mask);
Function pointers for pci_warn vs. pci_err ?
This looks like a lot of copy/paste.
Quoted 9 lines · Sinan Kaya
Quoted 4 lines · Matt Jolly
drivers/pci/pcie/aer.c +4
-0+pci_warn (dev," device [%04x:%04x] error status/mask=%08x/%08x\n" , + dev->vendor, dev->device, + info->status, info->mask); + }else {<snip>
Quoted 5 lines
drivers/pci/pcie/aer.c void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) · line 710
Quoted 3 lines
drivers/pci/pcie/aer.c +3
-0+pci_err (dev," device [%04x:%04x] error status/mask=%08x/%08x\n" , + dev->vendor, dev->device, + info->status, info->mask);Function pointers for
pci_warnvs.pci_err?
Not really possible as both are function-like macros.
Quoted 25 lines · Matt Jolly
The AER documentation indicates that correctable (severity=Corrected) errors should be output as a warning so that users can filter these errors if they choose to; This functionality does not appear to have been implemented.
This patch modifies the functions
aer_print_errorand__aer_print_error
to send correctable errors as a warning (pci_warn), rather than as an error (pci_err). It partially addresses several bugs in relation to kernel message buffer
spam for misbehaving devices - the root cause (possibly device firmware?) isn't addressed, but the dmesg output is less alarming for end users, and can
be filtered separately from uncorrectable errors. This should hopefully reduce the need for users to disable AER to suppress corrected errors.Signed-off-by Matt Jolly <[email protected]>--- drivers/pci/pcie/aer.c | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-)drivers/pci/pcie/aer.c +0
-0
drivers/pci/pcie/aer.c static void __aer_print_error(struct pci_dev *dev, · line 667
Quoted 10 lines
drivers/pci/pcie/aer.c +3
-4@@ -662,12 +662,18 @@ static void __aer_print_error(struct pci_dev *dev, errmsg = i <ARRAY_SIZE (aer_uncorrectable_error_string) ? aer_uncorrectable_error_string[i] :NULL ; -if (errmsg) -pci_err (dev," [%2d] %-22s%s\n" , i, errmsg, - info->first_error == i ?" (First)" :"" ); -else +if (errmsg) { +if (info->severity ==AER_CORRECTABLE ) { + pci_warn(dev, " [%2d] %-22s%s\n", i, errmsg,
I think we can use pci_printk() here to reduce the code duplication.
And I think we can also simplify the aer_correctable_error_string/ aer_uncorrectable_error_string stuff above, which would make this even simpler.
I'll respond to this with my proposal.
Quoted 45 lines
drivers/pci/pcie/aer.c +23
-6+ info->first_error == i ? " (First)" : ""); + }else { +pci_err (dev," [%2d] %-22s%s\n" , i, errmsg, + info->first_error == i ?" (First)" :"" ); + } + }else {pci_err (dev," [%2d] Unknown Error Bit%s\n" , i, info->first_error == i ?" (First)" :"" ); + } }pci_dev_aer_stats_incr (dev, info); }@@ -686,13 +692,23 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) layer =AER_GET_LAYER_ERROR (info->severity, info->status); agent =AER_GET_AGENT (info->severity, info->status); -pci_err (dev,"PCIe Bus Error: severity=%s, type=%s, (%s)\n" , - aer_error_severity_string[info->severity], - aer_error_layer[layer], aer_agent_string[agent]); +if (info->severity ==AER_CORRECTABLE ) { +pci_warn (dev,"PCIe Bus Error: severity=%s, type=%s, (%s)\n" , + aer_error_severity_string[info->severity], + aer_error_layer[layer], aer_agent_string[agent]); -pci_err (dev," device [%04x:%04x] error status/mask=%08x/%08x\n" , - dev->vendor, dev->device, - info->status, info->mask); +pci_warn (dev," device [%04x:%04x] error status/mask=%08x/%08x\n" , + dev->vendor, dev->device, + info->status, info->mask); + }else { +pci_err (dev,"PCIe Bus Error: severity=%s, type=%s, (%s)\n" , + aer_error_severity_string[info->severity], + aer_error_layer[layer], aer_agent_string[agent]); + +pci_err (dev," device [%04x:%04x] error status/mask=%08x/%08x\n" , + dev->vendor, dev->device, + info->status, info->mask); + }__aer_print_error (dev, info);Signature · 1 line
2.26.2
PATCH 1/2 PCI/AER: Simplify __aer_print_error()
In reply to Matt JollyFrom: Bjorn Helgaas <[email protected]>
aer_correctable_error_string[] and aer_uncorrectable_error_string[] have descriptions of AER error status bits. Add NULL entries to these tables so all entries for bits 0-31 are defined. Then we don't have to check for ARRAY_SIZE() when decoding a status word, which simplifies __aer_print_error().
--- drivers/pci/pcie/aer.c | 48 ++++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 14 deletions(-)
drivers/pci/pcie/aer.c +34 -14
@@ -447,7 +447,7 @@ static const char *aer_error_layer[] = {"Transaction Layer" }; -static const char *aer_correctable_error_string[AER_MAX_TYPEOF_COR_ERRS ] = { +static const char *aer_correctable_error_string[] = {"RxErr" ,/* Bit Position 0 */ NULL ,NULL ,
@@ -464,9 +464,25 @@ static const char *aer_correctable_error_string[AER_MAX_TYPEOF_COR_ERRS] = {"NonFatalErr" ,/* Bit Position 13 */ "CorrIntErr" ,/* Bit Position 14 */ "HeaderOF" ,/* Bit Position 15 */ +NULL ,/* Bit Position 16 */ +NULL ,/* Bit Position 17 */ +NULL ,/* Bit Position 18 */ +NULL ,/* Bit Position 19 */ +NULL ,/* Bit Position 20 */ +NULL ,/* Bit Position 21 */ +NULL ,/* Bit Position 22 */ +NULL ,/* Bit Position 23 */ +NULL ,/* Bit Position 24 */ +NULL ,/* Bit Position 25 */ +NULL ,/* Bit Position 26 */ +NULL ,/* Bit Position 27 */ +NULL ,/* Bit Position 28 */ +NULL ,/* Bit Position 29 */ +NULL ,/* Bit Position 30 */ + NULL, /* Bit Position 31 */ }; -static const char *aer_uncorrectable_error_string[AER_MAX_TYPEOF_UNCOR_ERRS ] = { +static const char *aer_uncorrectable_error_string[] = {"Undefined" ,/* Bit Position 0 */ NULL , NULL,
@@ -494,6 +510,11 @@ static const char *aer_uncorrectable_error_string[AER_MAX_TYPEOF_UNCOR_ERRS] = {"AtomicOpBlocked" ,/* Bit Position 24 */ "TLPBlockedErr" ,/* Bit Position 25 */ "PoisonTLPBlocked" ,/* Bit Position 26 */ +NULL ,/* Bit Position 27 */ +NULL ,/* Bit Position 28 */ +NULL ,/* Bit Position 29 */ +NULL ,/* Bit Position 30 */ + NULL, /* Bit Position 31 */ }; static const char *aer_agent_string[] = {
@@ -650,24 +671,23 @@ static void __print_tlp_header(struct pci_dev *dev,static void __aer_print_error (struct pci_dev *dev,struct aer_err_info *info) { +const char **strings;unsigned long status = info->status & ~info->mask; -const char *errmsg =NULL ; +const char *errmsg;int i; +if (info->severity ==AER_CORRECTABLE ) + strings = aer_correctable_error_string; +else + strings = aer_uncorrectable_error_string; +for_each_set_bit (i, &status,32 ) { -if (info->severity ==AER_CORRECTABLE ) - errmsg = i <ARRAY_SIZE (aer_correctable_error_string) ? - aer_correctable_error_string[i] :NULL ; -else - errmsg = i <ARRAY_SIZE (aer_uncorrectable_error_string) ? - aer_uncorrectable_error_string[i] :NULL ; + errmsg = strings[i]; +if (!errmsg) + errmsg ="Unknown Error Bit" ; -if (errmsg) -pci_err (dev," [%2d] %-22s%s\n" , i, errmsg, +pci_err (dev," [%2d] %-22s%s\n" , i, errmsg, info->first_error == i ?" (First)" :"" ); -else -pci_err (dev," [%2d] Unknown Error Bit%s\n" , - i, info->first_error == i ?" (First)" :"" ); }pci_dev_aer_stats_incr (dev, info); }
Signature · 1 line
2.25.1
PATCH 2/2 PCI/AER: Log correctable errors as warning, not error
From: Matt Jolly <[email protected]>
PCIe correctable errors are recovered by hardware with no need for software intervention (PCIe r5.0, sec 6.2.2.1).
Reduce the log level of correctable errors from KERN_ERR to KERN_WARNING.
The bug reports below are for correctable error logging. This doesn't fix the cause of those reports, but it may make the messages less alarming.
--- drivers/pci/pcie/aer.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-)
drivers/pci/pcie/aer.c +15 -10
@@ -673,20 +673,23 @@ static void __aer_print_error(struct pci_dev *dev, {const char **strings;unsigned long status = info->status & ~info->mask; -const char *errmsg; +const char *level, *errmsg;int i; -if (info->severity ==AER_CORRECTABLE ) +if (info->severity ==AER_CORRECTABLE ) { strings = aer_correctable_error_string; -else + level =KERN_WARNING ; + }else { strings = aer_uncorrectable_error_string; + level =KERN_ERR ; + }for_each_set_bit (i, &status,32 ) { errmsg = strings[i];if (!errmsg) errmsg ="Unknown Error Bit" ; -pci_err (dev," [%2d] %-22s%s\n" , i, errmsg, +pci_printk (level, dev," [%2d] %-22s%s\n" , i, errmsg, info->first_error == i ?" (First)" :"" ); }pci_dev_aer_stats_incr (dev, info);
@@ -696,6 +699,7 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) {int layer, agent;int id = ((dev->bus->number <<8 ) | dev->devfn); +const char *level;if (!info->status) { pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n",
@@ -706,13 +710,14 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) layer =AER_GET_LAYER_ERROR (info->severity, info->status); agent =AER_GET_AGENT (info->severity, info->status); -pci_err (dev,"PCIe Bus Error: severity=%s, type=%s, (%s)\n" , - aer_error_severity_string[info->severity], - aer_error_layer[layer], aer_agent_string[agent]); + level = (info->severity ==AER_CORRECTABLE ) ?KERN_WARNING :KERN_ERR ; + +pci_printk (level, dev,"PCIe Bus Error: severity=%s, type=%s, (%s)\n" , + aer_error_severity_string[info->severity], + aer_error_layer[layer], aer_agent_string[agent]); -pci_err (dev," device [%04x:%04x] error status/mask=%08x/%08x\n" , - dev->vendor, dev->device, - info->status, info->mask); +pci_printk (level, dev," device [%04x:%04x] error status/mask=%08x/%08x\n" , + dev->vendor, dev->device, info->status, info->mask);__aer_print_error (dev, info);
Signature · 1 line
2.25.1
Bjorn Helgaas
AppliedRe: [PATCH 2/2] PCI/AER: Log correctable errors as warning, not error
Quoted 10 lines · Bjorn Helgaas
From: Matt Jolly <[email protected]>
PCIe correctable errors are recovered by hardware with no need for software intervention (PCIe r5.0, sec 6.2.2.1).
Reduce the log level of correctable errors from
KERN_ERRtoKERN_WARNING.The bug reports below are for correctable error logging. This doesn't fix the cause of those reports, but it may make the messages less alarming.
Quoted 6 lines
[bhelgaas: commit log, use pci_printk() to avoid code duplication]Signed-off-by Matt Jolly <[email protected]>Signed-off-by Bjorn Helgaas <[email protected]>
I applied both of these to pci/error for v5.9.
Quoted 68 lines
--- drivers/pci/pcie/aer.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-)drivers/pci/pcie/aer.c +15
-10@@ -673,20 +673,23 @@ static void __aer_print_error(struct pci_dev *dev, {const char **strings;unsigned long status = info->status & ~info->mask; -const char *errmsg; +const char *level, *errmsg;int i; -if (info->severity ==AER_CORRECTABLE ) +if (info->severity ==AER_CORRECTABLE ) { strings = aer_correctable_error_string; -else + level =KERN_WARNING ; + }else { strings = aer_uncorrectable_error_string; + level =KERN_ERR ; + }for_each_set_bit (i, &status,32 ) { errmsg = strings[i];if (!errmsg) errmsg ="Unknown Error Bit" ; -pci_err (dev," [%2d] %-22s%s\n" , i, errmsg, +pci_printk (level, dev," [%2d] %-22s%s\n" , i, errmsg, info->first_error == i ?" (First)" :"" ); }pci_dev_aer_stats_incr (dev, info);@@ -696,6 +699,7 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) {int layer, agent;int id = ((dev->bus->number <<8 ) | dev->devfn); +const char *level;if (!info->status) { pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n",@@ -706,13 +710,14 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) layer =AER_GET_LAYER_ERROR (info->severity, info->status); agent =AER_GET_AGENT (info->severity, info->status); -pci_err (dev,"PCIe Bus Error: severity=%s, type=%s, (%s)\n" , - aer_error_severity_string[info->severity], - aer_error_layer[layer], aer_agent_string[agent]); + level = (info->severity ==AER_CORRECTABLE ) ?KERN_WARNING :KERN_ERR ; + +pci_printk (level, dev,"PCIe Bus Error: severity=%s, type=%s, (%s)\n" , + aer_error_severity_string[info->severity], + aer_error_layer[layer], aer_agent_string[agent]); -pci_err (dev," device [%04x:%04x] error status/mask=%08x/%08x\n" , - dev->vendor, dev->device, - info->status, info->mask); +pci_printk (level, dev," device [%04x:%04x] error status/mask=%08x/%08x\n" , + dev->vendor, dev->device, info->status, info->mask);__aer_print_error (dev, info);Signature · 1 line
2.25.1