EEH Support for PHB3
Subsystems (2)
cover EEH Support for PHB3
Naturally, EEH has been supported for PHB3, but we had some mask bits to disable it because the firmware isn't ready. The series of patch instends to remove those "mask bits" and enable EEH for PHB3. Besides, the output messages from EEH has been reordered to reflect the correct steps during EEH recovery. Also, we have dedicated data struct for PHB3 diag-data, which is similiar to the case of P7IOC.
Note: We can't recover fenced PHB3 this moment because the f/w still have some problems, which I'm tracing down. However, the fix shouldn't affect the logic we have in Linux side.
In order to force frozen PE, you need issue following command or similiar one on different PHB#:
echo 0x8000000000000000 > /sys/kernel/debug/powerpc/PCI0003/err_injct_outbound
---
arch/powerpc/include/asm/opal.h | 65 +++++++++++ arch/powerpc/kernel/eeh.c | 6 +- arch/powerpc/platforms/powernv/eeh-ioda.c | 153 ++++++++++++++++++++++---- arch/powerpc/platforms/powernv/eeh-powernv.c | 5 +- arch/powerpc/platforms/powernv/pci.h | 2 +- 5 files changed, 200 insertions(+), 31 deletions(-)
Thanks,
Gavin
PATCH 1/6 powerpc/powernv: Enable EEH for PHB3
The EEH isn't enabled for PHB3 and the patch intends to enable it.
--- arch/powerpc/platforms/powernv/eeh-ioda.c | 26 +++++++++++--------------- arch/powerpc/platforms/powernv/eeh-powernv.c | 5 +---- 2 files changed, 12 insertions(+), 19 deletions(-)
arch/powerpc/platforms/powernv/eeh-ioda.c +11 -15
@@ -113,27 +113,23 @@ static int ioda_eeh_post_init(struct pci_controller *hose) ioda_eeh_nb_init =1 ; } -/* FIXME: Enable it for PHB3 later */ -if (phb->type ==PNV_PHB_IODA1 ) { +/* We needn't HUB diag-data on PHB3 */ +if (phb->type ==PNV_PHB_IODA1 && !hub_diag) { + hub_diag = (char *)__get_free_page (GFP_KERNEL | __GFP_ZERO);if (!hub_diag) { - hub_diag = (char *)__get_free_page (GFP_KERNEL | - __GFP_ZERO); -if (!hub_diag) { -pr_err ("%s: Out of memory !\n" , - __func__); -return -ENOMEM ; - } +pr_err ("%s: Out of memory !\n" , __func__); +return -ENOMEM ; } + }#ifdef CONFIG_DEBUG_FS -if (phb->dbgfs) -debugfs_create_file ("err_injct" ,0600 , - phb->dbgfs, hose, - &ioda_eeh_dbgfs_ops); +if (phb->dbgfs) +debugfs_create_file ("err_injct" ,0600 , + phb->dbgfs, hose, + &ioda_eeh_dbgfs_ops);#endif - phb->eeh_state |=PNV_EEH_STATE_ENABLED ; - } + phb->eeh_state |=PNV_EEH_STATE_ENABLED ;return 0 ; }
arch/powerpc/platforms/powernv/eeh-powernv.c +1 -4
@@ -144,11 +144,8 @@ static int powernv_eeh_dev_probe(struct pci_dev *dev, void *flag)/* * Enable EEH explicitly so that we will do EEH check * while accessing I/O stuff -* -* FIXME: Enable that for PHB3 later */ -if (phb->type ==PNV_PHB_IODA1 ) - eeh_subsystem_enabled =1 ; + eeh_subsystem_enabled =1 ;/* Save memory bars */ eeh_save_bars (edev);
Signature · 1 line
1.7.5.4
PATCH 2/6 powerpc/powernv: Support inbound error injection
In reply to Gavin ShanFor now, we only support outbound error injection. Actually, the hardware supports injecting inbound errors as well. The patch enables to inject inbound errors.
--- arch/powerpc/platforms/powernv/eeh-ioda.c | 59 ++++++++++++++++++++++++---- 1 files changed, 50 insertions(+), 9 deletions(-)
arch/powerpc/platforms/powernv/eeh-ioda.c +50 -9
@@ -66,26 +66,60 @@ static struct notifier_block ioda_eeh_nb = { };#ifdef CONFIG_DEBUG_FS -static int ioda_eeh_dbgfs_set (void *data,u64 val) +static int ioda_eeh_dbgfs_set (void *data,int offset,u64 val) {struct pci_controller *hose = data;struct pnv_phb *phb = hose->private_data; -out_be64 (phb->regs +0xD10 , val); +out_be64 (phb->regs + offset, val);return 0 ; } -static int ioda_eeh_dbgfs_get (void *data,u64 *val) +static int ioda_eeh_dbgfs_get (void *data,int offset,u64 *val) {struct pci_controller *hose = data;struct pnv_phb *phb = hose->private_data; - *val =in_be64 (phb->regs +0xD10 ); + *val =in_be64 (phb->regs + offset);return 0 ; } -DEFINE_SIMPLE_ATTRIBUTE (ioda_eeh_dbgfs_ops, ioda_eeh_dbgfs_get, - ioda_eeh_dbgfs_set,"0x%llx\n" ); +static int ioda_eeh_outb_dbgfs_set (void *data,u64 val) +{ +return ioda_eeh_dbgfs_set (data,0xD10 , val); +} + +static int ioda_eeh_outb_dbgfs_get (void *data,u64 *val) +{ +return ioda_eeh_dbgfs_get (data,0xD10 , val); +} + +static int ioda_eeh_inbA_dbgfs_set (void *data,u64 val) +{ +return ioda_eeh_dbgfs_set (data,0xD90 , val); +} + +static int ioda_eeh_inbA_dbgfs_get (void *data,u64 *val) +{ +return ioda_eeh_dbgfs_get (data,0xD90 , val); +} + +static int ioda_eeh_inbB_dbgfs_set (void *data,u64 val) +{ +return ioda_eeh_dbgfs_set (data,0xE10 , val); +} + +static int ioda_eeh_inbB_dbgfs_get (void *data,u64 *val) +{ +return ioda_eeh_dbgfs_get (data,0xE10 , val); +} + +DEFINE_SIMPLE_ATTRIBUTE (ioda_eeh_outb_dbgfs_ops, ioda_eeh_outb_dbgfs_get, + ioda_eeh_outb_dbgfs_set,"0x%llx\n" ); +DEFINE_SIMPLE_ATTRIBUTE (ioda_eeh_inbA_dbgfs_ops, ioda_eeh_inbA_dbgfs_get, + ioda_eeh_inbA_dbgfs_set,"0x%llx\n" ); +DEFINE_SIMPLE_ATTRIBUTE (ioda_eeh_inbB_dbgfs_ops, ioda_eeh_inbB_dbgfs_get, + ioda_eeh_inbB_dbgfs_set,"0x%llx\n" );#endif /* CONFIG_DEBUG_FS */ /**
@@ -123,10 +157,17 @@ static int ioda_eeh_post_init(struct pci_controller *hose) }#ifdef CONFIG_DEBUG_FS -if (phb->dbgfs) -debugfs_create_file ("err_injct" ,0600 , +if (phb->dbgfs) { +debugfs_create_file ("err_injct_outbound" ,0600 , + phb->dbgfs, hose, + &ioda_eeh_outb_dbgfs_ops); +debugfs_create_file ("err_injct_inboundA" ,0600 , phb->dbgfs, hose, - &ioda_eeh_dbgfs_ops); + &ioda_eeh_inbA_dbgfs_ops); +debugfs_create_file ("err_injct_inboundB" ,0600 , + phb->dbgfs, hose, + &ioda_eeh_inbB_dbgfs_ops); + }#endif phb->eeh_state |=PNV_EEH_STATE_ENABLED ;
Signature · 1 line
1.7.5.4
PATCH 3/6 powerpc/eeh: Output error number
In reply to Gavin ShanThe patch prints the error number while failing to retrieve error log from firmware. It's helpful for debugging.
--- arch/powerpc/platforms/powernv/eeh-ioda.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
arch/powerpc/platforms/powernv/eeh-ioda.c +2 -2
@@ -590,8 +590,8 @@ static int ioda_eeh_get_log(struct eeh_pe *pe, int severity, phb->diag.blob, PNV_PCI_DIAG_BUF_SIZE);if (ret) {spin_unlock_irqrestore (&phb->lock, flags); -pr_warning ("%s: Failed to get log for PHB#%x-PE#%x\n" , - __func__, hose->global_number, pe->addr); +pr_warning ("%s: Can't get log for PHB#%x-PE#%x (%lld)\n" , + __func__, hose->global_number, pe->addr, ret);return -EIO ; }
Signature · 1 line
1.7.5.4
PATCH 4/6 powerpc/powernv: Double size of log blob
In reply to Gavin ShanEach PHB instance (struct pnv_phb) has its corresponding log blob, which is used to hold the retrieved error log from firmware. The current size of that (4096) isn't enough for PHB3 case and the patch makes that double to 8192.
--- arch/powerpc/platforms/powernv/pci.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)
arch/powerpc/platforms/powernv/pci.h +1 -1
@@ -17,7 +17,7 @@ enum pnv_phb_model { PNV_PHB_MODEL_PHB3, }; -#define PNV_PCI_DIAG_BUF_SIZE 4096 +#define PNV_PCI_DIAG_BUF_SIZE 8192#define PNV_IODA_PE_DEV (1 << 0)/* PE has single PCI device */ #define PNV_IODA_PE_BUS (1 << 1)/* PE has primary PCI bus */ #define PNV_IODA_PE_BUS_ALL (1 << 2)/* PE has subordinate buses */
Signature · 1 line
1.7.5.4
PATCH 5/6 powerpc/eeh: Output PHB3 diag-data
In reply to Gavin ShanThe patch adds function ioda_eeh_phb3_phb_diag() to dump PHB3 PHB diag-data. That's called while detecting informative errors or frozen PE on the specific PHB.
--- arch/powerpc/include/asm/opal.h | 65 ++++++++++++++++++++++++++ arch/powerpc/platforms/powernv/eeh-ioda.c | 70 +++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 0 deletions(-)
arch/powerpc/include/asm/opal.h +65 -0
@@ -444,10 +444,12 @@ enum {enum {OPAL_PHB_ERROR_DATA_TYPE_P7IOC =1 , +OPAL_PHB_ERROR_DATA_TYPE_PHB3 =2 };enum {OPAL_P7IOC_NUM_PEST_REGS =128 , +OPAL_PHB3_NUM_PEST_REGS =256 };struct OpalIoPhbErrorCommon {
@@ -515,6 +517,69 @@ struct OpalIoP7IOCPhbErrorData {uint64_t pestB[OPAL_P7IOC_NUM_PEST_REGS ]; }; +struct OpalIoPhb3ErrorData { +struct OpalIoPhbErrorCommon common; + +uint32_t brdgCtl; + +/* PHB3 UTL regs */ +uint32_t portStatusReg; +uint32_t rootCmplxStatus; +uint32_t busAgentStatus; + +/* PHB3 cfg regs */ +uint32_t deviceStatus; +uint32_t slotStatus; +uint32_t linkStatus; +uint32_t devCmdStatus; +uint32_t devSecStatus; + +/* cfg AER regs */ +uint32_t rootErrorStatus; +uint32_t uncorrErrorStatus; +uint32_t corrErrorStatus; +uint32_t tlpHdr1; +uint32_t tlpHdr2; +uint32_t tlpHdr3; +uint32_t tlpHdr4; +uint32_t sourceId; + +uint32_t rsv3; + +/* Record data about the call to allocate a buffer */ +uint64_t errorClass; +uint64_t correlator; + +uint64_t nFir;/* 000 */ +uint64_t nFirMask;/* 003 */ +uint64_t nFirWOF;/* 008 */ + +/* PHB3 MMIO Error Regs */ +uint64_t phbPlssr;/* 120 */ +uint64_t phbCsr;/* 110 */ +uint64_t lemFir;/* C00 */ +uint64_t lemErrorMask;/* C18 */ +uint64_t lemWOF;/* C40 */ +uint64_t phbErrorStatus;/* C80 */ +uint64_t phbFirstErrorStatus;/* C88 */ +uint64_t phbErrorLog0;/* CC0 */ +uint64_t phbErrorLog1;/* CC8 */ +uint64_t mmioErrorStatus;/* D00 */ +uint64_t mmioFirstErrorStatus;/* D08 */ +uint64_t mmioErrorLog0;/* D40 */ +uint64_t mmioErrorLog1;/* D48 */ +uint64_t dma0ErrorStatus;/* D80 */ +uint64_t dma0FirstErrorStatus;/* D88 */ +uint64_t dma0ErrorLog0;/* DC0 */ +uint64_t dma0ErrorLog1;/* DC8 */ +uint64_t dma1ErrorStatus;/* E00 */ +uint64_t dma1FirstErrorStatus;/* E08 */ +uint64_t dma1ErrorLog0;/* E40 */ +uint64_t dma1ErrorLog1;/* E48 */ +uint64_t pestA[OPAL_PHB3_NUM_PEST_REGS ]; +uint64_t pestB[OPAL_PHB3_NUM_PEST_REGS ]; +}; + typedef struct oppanel_line { const char * line; uint64_t line_len;
arch/powerpc/platforms/powernv/eeh-ioda.c +70 -0
82 lines not on this page · raw message
Signature · 1 line
1.7.5.4
PATCH 6/6 powerpc/eeh: Reorder output messages
In reply to Gavin ShanWe already had some output messages from EEH core. Occasionally, we can see the output messages from EEH core before the stack dump. That's not what we expected. The patch fixes that and shows the stack dump prior to output messages from EEH core.
--- arch/powerpc/kernel/eeh.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-)
arch/powerpc/kernel/eeh.c +3 -3
@@ -327,11 +327,11 @@ static int eeh_phb_check_failure(struct eeh_pe *pe)/* Isolate the PHB and send event */ eeh_pe_state_mark (phb_pe,EEH_PE_ISOLATED );eeh_serialize_unlock (flags); -eeh_send_failure_event (phb_pe);pr_err ("EEH: PHB#%x failure detected\n" , phb_pe->phb->global_number);dump_stack (); +eeh_send_failure_event (phb_pe);return 1 ; out:
@@ -454,8 +454,6 @@ int eeh_dev_check_failure(struct eeh_dev *edev)eeh_pe_state_mark (pe,EEH_PE_ISOLATED );eeh_serialize_unlock (flags); -eeh_send_failure_event (pe); -/* Most EEH events are due to device driver bugs. Having * a stack trace will help the device-driver authors figure * out what happened. So print that out.
@@ -464,6 +462,8 @@ int eeh_dev_check_failure(struct eeh_dev *edev) pe->addr, pe->phb->global_number);dump_stack (); +eeh_send_failure_event (pe); +return 1 ; dn_unlock:
Signature · 1 line
1.7.5.4