Skip to contentSkip to conversation

EEH Support for PHB3

PATCH New Gavin Shan 1 participant 7 messages Sep 6, 2013

Subsystems (2)
lore
Share & tools
lore

Gavin Shan

author
#0 lore raw

cover EEH Support for PHB3

Naturally, EEH has been supported for PHB3, but we had some mask bits to disable it because the firmware isn't ready. The series of patch instends to remove those "mask bits" and enable EEH for PHB3. Besides, the output messages from EEH has been reordered to reflect the correct steps during EEH recovery. Also, we have dedicated data struct for PHB3 diag-data, which is similiar to the case of P7IOC.

Note: We can't recover fenced PHB3 this moment because the f/w still have some problems, which I'm tracing down. However, the fix shouldn't affect the logic we have in Linux side.

In order to force frozen PE, you need issue following command or similiar one on different PHB#:

echo 0x8000000000000000 > /sys/kernel/debug/powerpc/PCI0003/err_injct_outbound

---

arch/powerpc/include/asm/opal.h              |   65 +++++++++++
arch/powerpc/kernel/eeh.c                    |    6 +-
arch/powerpc/platforms/powernv/eeh-ioda.c    |  153 ++++++++++++++++++++++----
arch/powerpc/platforms/powernv/eeh-powernv.c |    5 +-
arch/powerpc/platforms/powernv/pci.h         |    2 +-
5 files changed, 200 insertions(+), 31 deletions(-)

Thanks,
Gavin

author
#1 lore raw

PATCH 1/6 powerpc/powernv: Enable EEH for PHB3

The EEH isn't enabled for PHB3 and the patch intends to enable it.

---
 arch/powerpc/platforms/powernv/eeh-ioda.c    |   26 +++++++++++---------------
 arch/powerpc/platforms/powernv/eeh-powernv.c |    5 +----
 2 files changed, 12 insertions(+), 19 deletions(-)
arch/powerpc/platforms/powernv/eeh-ioda.c +11 -15
index 0cd1c4a..855c1d5 100644
@@ -113,27 +113,23 @@ static int ioda_eeh_post_init(struct pci_controller *hose)
 		ioda_eeh_nb_init = 1;
 	}
 
-	/* FIXME: Enable it for PHB3 later */
-	if (phb->type == PNV_PHB_IODA1) {
+	/* We needn't HUB diag-data on PHB3 */
+	if (phb->type == PNV_PHB_IODA1 && !hub_diag) {
+		hub_diag = (char *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
 		if (!hub_diag) {
-			hub_diag = (char *)__get_free_page(GFP_KERNEL |
-							   __GFP_ZERO);
-			if (!hub_diag) {
-				pr_err("%s: Out of memory !\n",
-				       __func__);
-				return -ENOMEM;
-			}
+			pr_err("%s: Out of memory !\n", __func__);
+			return -ENOMEM;
 		}
+	}
 
 #ifdef CONFIG_DEBUG_FS
-		if (phb->dbgfs)
-			debugfs_create_file("err_injct", 0600,
-					    phb->dbgfs, hose,
-					    &ioda_eeh_dbgfs_ops);
+	if (phb->dbgfs)
+		debugfs_create_file("err_injct", 0600,
+				    phb->dbgfs, hose,
+				    &ioda_eeh_dbgfs_ops);
 #endif
 
-		phb->eeh_state |= PNV_EEH_STATE_ENABLED;
-	}
+	phb->eeh_state |= PNV_EEH_STATE_ENABLED;
 
 	return 0;
 }
arch/powerpc/platforms/powernv/eeh-powernv.c +1 -4
index 79663d2..73b9814 100644
@@ -144,11 +144,8 @@ static int powernv_eeh_dev_probe(struct pci_dev *dev, void *flag)
 	/*
 	 * Enable EEH explicitly so that we will do EEH check
 	 * while accessing I/O stuff
-	 *
-	 * FIXME: Enable that for PHB3 later
 	 */
-	if (phb->type == PNV_PHB_IODA1)
-		eeh_subsystem_enabled = 1;
+	eeh_subsystem_enabled = 1;
 
 	/* Save memory bars */
 	eeh_save_bars(edev);
Signature · 1 line
1.7.5.4
author
#2 lore raw

PATCH 2/6 powerpc/powernv: Support inbound error injection

In reply to Gavin Shan

For now, we only support outbound error injection. Actually, the hardware supports injecting inbound errors as well. The patch enables to inject inbound errors.

---
 arch/powerpc/platforms/powernv/eeh-ioda.c |   59 ++++++++++++++++++++++++----
 1 files changed, 50 insertions(+), 9 deletions(-)
arch/powerpc/platforms/powernv/eeh-ioda.c +50 -9
index 855c1d5..3ee44b0 100644
@@ -66,26 +66,60 @@ static struct notifier_block ioda_eeh_nb = {
 };
 
 #ifdef CONFIG_DEBUG_FS
-static int ioda_eeh_dbgfs_set(void *data, u64 val)
+static int ioda_eeh_dbgfs_set(void *data, int offset, u64 val)
 {
 	struct pci_controller *hose = data;
 	struct pnv_phb *phb = hose->private_data;
 
-	out_be64(phb->regs + 0xD10, val);
+	out_be64(phb->regs + offset, val);
 	return 0;
 }
 
-static int ioda_eeh_dbgfs_get(void *data, u64 *val)
+static int ioda_eeh_dbgfs_get(void *data, int offset, u64 *val)
 {
 	struct pci_controller *hose = data;
 	struct pnv_phb *phb = hose->private_data;
 
-	*val = in_be64(phb->regs + 0xD10);
+	*val = in_be64(phb->regs + offset);
 	return 0;
 }
 
-DEFINE_SIMPLE_ATTRIBUTE(ioda_eeh_dbgfs_ops, ioda_eeh_dbgfs_get,
-			ioda_eeh_dbgfs_set, "0x%llx\n");
+static int ioda_eeh_outb_dbgfs_set(void *data, u64 val)
+{
+	return ioda_eeh_dbgfs_set(data, 0xD10, val);
+}
+
+static int ioda_eeh_outb_dbgfs_get(void *data, u64 *val)
+{
+	return ioda_eeh_dbgfs_get(data, 0xD10, val);
+}
+
+static int ioda_eeh_inbA_dbgfs_set(void *data, u64 val)
+{
+	return ioda_eeh_dbgfs_set(data, 0xD90, val);
+}
+
+static int ioda_eeh_inbA_dbgfs_get(void *data, u64 *val)
+{
+	return ioda_eeh_dbgfs_get(data, 0xD90, val);
+}
+
+static int ioda_eeh_inbB_dbgfs_set(void *data, u64 val)
+{
+	return ioda_eeh_dbgfs_set(data, 0xE10, val);
+}
+
+static int ioda_eeh_inbB_dbgfs_get(void *data, u64 *val)
+{
+	return ioda_eeh_dbgfs_get(data, 0xE10, val);
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(ioda_eeh_outb_dbgfs_ops, ioda_eeh_outb_dbgfs_get,
+			ioda_eeh_outb_dbgfs_set, "0x%llx\n");
+DEFINE_SIMPLE_ATTRIBUTE(ioda_eeh_inbA_dbgfs_ops, ioda_eeh_inbA_dbgfs_get,
+			ioda_eeh_inbA_dbgfs_set, "0x%llx\n");
+DEFINE_SIMPLE_ATTRIBUTE(ioda_eeh_inbB_dbgfs_ops, ioda_eeh_inbB_dbgfs_get,
+			ioda_eeh_inbB_dbgfs_set, "0x%llx\n");
 #endif /* CONFIG_DEBUG_FS */
 
 /**
@@ -123,10 +157,17 @@ static int ioda_eeh_post_init(struct pci_controller *hose)
 	}
 
 #ifdef CONFIG_DEBUG_FS
-	if (phb->dbgfs)
-		debugfs_create_file("err_injct", 0600,
+	if (phb->dbgfs) {
+		debugfs_create_file("err_injct_outbound", 0600,
+				    phb->dbgfs, hose,
+				    &ioda_eeh_outb_dbgfs_ops);
+		debugfs_create_file("err_injct_inboundA", 0600,
 				    phb->dbgfs, hose,
-				    &ioda_eeh_dbgfs_ops);
+				    &ioda_eeh_inbA_dbgfs_ops);
+		debugfs_create_file("err_injct_inboundB", 0600,
+				    phb->dbgfs, hose,
+				    &ioda_eeh_inbB_dbgfs_ops);
+	}
 #endif
 
 	phb->eeh_state |= PNV_EEH_STATE_ENABLED;
Signature · 1 line
1.7.5.4
author
#3 lore raw

PATCH 3/6 powerpc/eeh: Output error number

In reply to Gavin Shan

The patch prints the error number while failing to retrieve error log from firmware. It's helpful for debugging.

---
 arch/powerpc/platforms/powernv/eeh-ioda.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)
arch/powerpc/platforms/powernv/eeh-ioda.c +2 -2
index 3ee44b0..d60ba3b 100644
@@ -590,8 +590,8 @@ static int ioda_eeh_get_log(struct eeh_pe *pe, int severity,
 			phb->diag.blob, PNV_PCI_DIAG_BUF_SIZE);
 	if (ret) {
 		spin_unlock_irqrestore(&phb->lock, flags);
-		pr_warning("%s: Failed to get log for PHB#%x-PE#%x\n",
-			   __func__, hose->global_number, pe->addr);
+		pr_warning("%s: Can't get log for PHB#%x-PE#%x (%lld)\n",
+			   __func__, hose->global_number, pe->addr, ret);
 		return -EIO;
 	}
 
Signature · 1 line
1.7.5.4
author
#4 lore raw

PATCH 4/6 powerpc/powernv: Double size of log blob

In reply to Gavin Shan

Each PHB instance (struct pnv_phb) has its corresponding log blob, which is used to hold the retrieved error log from firmware. The current size of that (4096) isn't enough for PHB3 case and the patch makes that double to 8192.

---
 arch/powerpc/platforms/powernv/pci.h |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
arch/powerpc/platforms/powernv/pci.h +1 -1
index d633c64..f4bccc8 100644
@@ -17,7 +17,7 @@ enum pnv_phb_model {
 	PNV_PHB_MODEL_PHB3,
 };
 
-#define PNV_PCI_DIAG_BUF_SIZE	4096
+#define PNV_PCI_DIAG_BUF_SIZE	8192
 #define PNV_IODA_PE_DEV		(1 << 0)	/* PE has single PCI device	*/
 #define PNV_IODA_PE_BUS		(1 << 1)	/* PE has primary PCI bus	*/
 #define PNV_IODA_PE_BUS_ALL	(1 << 2)	/* PE has subordinate buses	*/
Signature · 1 line
1.7.5.4
author
#5 lore raw

PATCH 5/6 powerpc/eeh: Output PHB3 diag-data

In reply to Gavin Shan

The patch adds function ioda_eeh_phb3_phb_diag() to dump PHB3 PHB diag-data. That's called while detecting informative errors or frozen PE on the specific PHB.

---
 arch/powerpc/include/asm/opal.h           |   65 ++++++++++++++++++++++++++
 arch/powerpc/platforms/powernv/eeh-ioda.c |   70 +++++++++++++++++++++++++++++
 2 files changed, 135 insertions(+), 0 deletions(-)
arch/powerpc/include/asm/opal.h +65 -0
index 029fe85..fed4b7d 100644
@@ -444,10 +444,12 @@ enum {
 
 enum {
 	OPAL_PHB_ERROR_DATA_TYPE_P7IOC = 1,
+	OPAL_PHB_ERROR_DATA_TYPE_PHB3 = 2
 };
 
 enum {
 	OPAL_P7IOC_NUM_PEST_REGS = 128,
+	OPAL_PHB3_NUM_PEST_REGS = 256
 };
 
 struct OpalIoPhbErrorCommon {
@@ -515,6 +517,69 @@ struct OpalIoP7IOCPhbErrorData {
 	uint64_t pestB[OPAL_P7IOC_NUM_PEST_REGS];
 };
 
+struct OpalIoPhb3ErrorData {
+	struct OpalIoPhbErrorCommon common;
+
+	uint32_t brdgCtl;
+
+	/* PHB3 UTL regs */
+	uint32_t portStatusReg;
+	uint32_t rootCmplxStatus;
+	uint32_t busAgentStatus;
+
+	/* PHB3 cfg regs */
+	uint32_t deviceStatus;
+	uint32_t slotStatus;
+	uint32_t linkStatus;
+	uint32_t devCmdStatus;
+	uint32_t devSecStatus;
+
+	/* cfg AER regs */
+	uint32_t rootErrorStatus;
+	uint32_t uncorrErrorStatus;
+	uint32_t corrErrorStatus;
+	uint32_t tlpHdr1;
+	uint32_t tlpHdr2;
+	uint32_t tlpHdr3;
+	uint32_t tlpHdr4;
+	uint32_t sourceId;
+
+	uint32_t rsv3;
+
+	/* Record data about the call to allocate a buffer */
+	uint64_t errorClass;
+	uint64_t correlator;
+
+	uint64_t nFir;			/* 000 */
+	uint64_t nFirMask;		/* 003 */
+	uint64_t nFirWOF;		/* 008 */
+
+	/* PHB3 MMIO Error Regs */
+	uint64_t phbPlssr;		/* 120 */
+	uint64_t phbCsr;		/* 110 */
+	uint64_t lemFir;		/* C00 */
+	uint64_t lemErrorMask;		/* C18 */
+	uint64_t lemWOF;		/* C40 */
+	uint64_t phbErrorStatus;	/* C80 */
+	uint64_t phbFirstErrorStatus;	/* C88 */
+	uint64_t phbErrorLog0;		/* CC0 */
+	uint64_t phbErrorLog1;		/* CC8 */
+	uint64_t mmioErrorStatus;	/* D00 */
+	uint64_t mmioFirstErrorStatus;	/* D08 */
+	uint64_t mmioErrorLog0;		/* D40 */
+	uint64_t mmioErrorLog1;		/* D48 */
+	uint64_t dma0ErrorStatus;	/* D80 */
+	uint64_t dma0FirstErrorStatus;	/* D88 */
+	uint64_t dma0ErrorLog0;		/* DC0 */
+	uint64_t dma0ErrorLog1;		/* DC8 */
+	uint64_t dma1ErrorStatus;	/* E00 */
+	uint64_t dma1FirstErrorStatus;	/* E08 */
+	uint64_t dma1ErrorLog0;		/* E40 */
+	uint64_t dma1ErrorLog1;		/* E48 */
+	uint64_t pestA[OPAL_PHB3_NUM_PEST_REGS];
+	uint64_t pestB[OPAL_PHB3_NUM_PEST_REGS];
+};
+
 typedef struct oppanel_line {
 	const char * 	line;
 	uint64_t 	line_len;
arch/powerpc/platforms/powernv/eeh-ioda.c +70 -0

82 lines not on this page · raw message

Signature · 1 line
1.7.5.4
author
#6 lore raw

PATCH 6/6 powerpc/eeh: Reorder output messages

In reply to Gavin Shan

We already had some output messages from EEH core. Occasionally, we can see the output messages from EEH core before the stack dump. That's not what we expected. The patch fixes that and shows the stack dump prior to output messages from EEH core.

---
 arch/powerpc/kernel/eeh.c |    6 +++---
 1 files changed, 3 insertions(+), 3 deletions(-)
arch/powerpc/kernel/eeh.c +3 -3
index 55593ee..1fb331d 100644
@@ -327,11 +327,11 @@ static int eeh_phb_check_failure(struct eeh_pe *pe)
 	/* Isolate the PHB and send event */
 	eeh_pe_state_mark(phb_pe, EEH_PE_ISOLATED);
 	eeh_serialize_unlock(flags);
-	eeh_send_failure_event(phb_pe);
 
 	pr_err("EEH: PHB#%x failure detected\n",
 		phb_pe->phb->global_number);
 	dump_stack();
+	eeh_send_failure_event(phb_pe);
 
 	return 1;
 out:
@@ -454,8 +454,6 @@ int eeh_dev_check_failure(struct eeh_dev *edev)
 	eeh_pe_state_mark(pe, EEH_PE_ISOLATED);
 	eeh_serialize_unlock(flags);
 
-	eeh_send_failure_event(pe);
-
 	/* Most EEH events are due to device driver bugs.  Having
 	 * a stack trace will help the device-driver authors figure
 	 * out what happened.  So print that out.
@@ -464,6 +462,8 @@ int eeh_dev_check_failure(struct eeh_dev *edev)
 		pe->addr, pe->phb->global_number);
 	dump_stack();
 
+	eeh_send_failure_event(pe);
+
 	return 1;
 
 dn_unlock:
Signature · 1 line
1.7.5.4