Skip to contentSkip to conversation

lpfc: Update lpfc to revision 14.2.0.1

PATCH Applied James Smart 2 participants 7 messages Mar 17, 2022 – Apr 7, 2022

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover lpfc: Update lpfc to revision 14.2.0.1

Update lpfc to revision 14.2.0.1

This patch set contains 3 fixes for errors detected during eeh error injection.

The patches were cut against Martin's 5.18/scsi-staging tree

James Smart (4):
lpfc: Improve PCI EEH Error and Recovery Handling
lpfc: Fix unload hang after back to back PCI EEH faults
lpfc: Fix queue failures when recovering from PCI parity error
lpfc: Update lpfc version to 14.2.0.1

 drivers/scsi/lpfc/lpfc.h         |   7 +-
 drivers/scsi/lpfc/lpfc_crtn.h    |   3 +
 drivers/scsi/lpfc/lpfc_hbadisc.c | 120 +++++++++++++++++++++++++------
 drivers/scsi/lpfc/lpfc_init.c    |  88 ++++++++++++++++-------
 drivers/scsi/lpfc/lpfc_nvme.c    |  27 ++++---
 drivers/scsi/lpfc/lpfc_sli.c     |  65 +++++++++++------
 drivers/scsi/lpfc/lpfc_version.h |   2 +-
 7 files changed, 232 insertions(+), 80 deletions(-)
Signature · 1 line
2.26.2
#1 lore raw

PATCH 1/4 lpfc: Improve PCI EEH Error and Recovery Handling

Following EEH errors, the driver can crash or hang when deleting the localport or when attempting to unload.

The EEH handlers in the driver did not notify the nvme-fc transport before tearing the driver down. This was delayed until the resume steps. THis worked for scsi because the lpfc_block_scsi would notify the scsi_fc_transport that the target was not available but it would not clean up all the references to the ndlp.

The sli3 prep for dev reset handler did the lpfc_offline_prep and lpfc_offline calls to get the port stopped before restarting. The sli4 version of the prep for dev reset just destroyed the queues and did not stop nvme from continuing. Also because the port was not really stopped the localport destroy would hang because the transport was still waiting for IO. Additionally, a Devloss tmo can fire and post events to a stopped worker thread creating another hang condition.

To Fix:
lpfc_sli4_prep_dev_for_reset is modified to call offline_prep and offline rather than just lpfc_scsi_dev_block to ensure both SCSI and NVME transports are notified to block I/O to the driver.

Logic is added to devloss handler and worker thread to clean up ndlp references and quiesce appropriately.

---
 drivers/scsi/lpfc/lpfc.h         |   7 +-
 drivers/scsi/lpfc/lpfc_crtn.h    |   3 +
 drivers/scsi/lpfc/lpfc_hbadisc.c | 119 +++++++++++++++++++++++++------
 drivers/scsi/lpfc/lpfc_init.c    |  60 ++++++++++------
 drivers/scsi/lpfc/lpfc_nvme.c    |  11 ++-
 drivers/scsi/lpfc/lpfc_sli.c     |  15 ++--
 6 files changed, 157 insertions(+), 58 deletions(-)
drivers/scsi/lpfc/lpfc.h +5 -2
index 86653aa9b389..8405fd0bbc59 100644
@@ -896,6 +896,11 @@ enum lpfc_irq_chann_mode {
 	NHT_MODE,
 };
 
+enum lpfc_hba_bit_flags {
+	FABRIC_COMANDS_BLOCKED,
+	HBA_PCI_ERR,
+};
+
 struct lpfc_hba {
 	/* SCSI interface function jump table entries */
 	struct lpfc_io_buf * (*lpfc_get_scsi_buf)
@@ -1042,7 +1047,6 @@ struct lpfc_hba {
 					 * Firmware supports Forced Link Speed
 					 * capability
 					 */
-#define HBA_PCI_ERR		0x80000 /* The PCI slot is offline */
 #define HBA_FLOGI_ISSUED	0x100000 /* FLOGI was issued */
 #define HBA_SHORT_CMF		0x200000 /* shorter CMF timer routine */
 #define HBA_CGN_DAY_WRAP	0x400000 /* HBA Congestion info day wraps */
@@ -1349,7 +1353,6 @@ struct lpfc_hba {
 	atomic_t fabric_iocb_count;
 	struct timer_list fabric_block_timer;
 	unsigned long bit_flags;
-#define	FABRIC_COMANDS_BLOCKED	0
 	atomic_t num_rsrc_err;
 	atomic_t num_cmd_success;
 	unsigned long last_rsrc_error_time;
drivers/scsi/lpfc/lpfc_crtn.h +3 -0
index 96408cd6c4c8..9897a1aa387b 100644
@@ -670,3 +670,6 @@ struct lpfc_vmid *lpfc_get_vmid_from_hashtable(struct lpfc_vport *vport,
 					      uint32_t hash, uint8_t *buf);
 void lpfc_vmid_vport_cleanup(struct lpfc_vport *vport);
 int lpfc_issue_els_qfpa(struct lpfc_vport *vport);
+
+void lpfc_sli_rpi_release(struct lpfc_vport *vport,
+			  struct lpfc_nodelist *ndlp);
drivers/scsi/lpfc/lpfc_hbadisc.c +96 -23

244 lines not on this page · raw message

drivers/scsi/lpfc/lpfc_init.c +38 -22

156 lines not on this page · raw message

drivers/scsi/lpfc/lpfc_nvme.c +5 -6
index 1213a299f9aa..e47205e0d3e2 100644
@@ -2169,8 +2169,7 @@ lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
 			abts_nvme = 0;
 			for (i = 0; i < phba->cfg_hdw_queue; i++) {
 				qp = &phba->sli4_hba.hdwq[i];
-				if (!vport || !vport->localport ||
-				    !qp || !qp->io_wq)
+				if (!vport->localport || !qp || !qp->io_wq)
 					return;
 
 				pring = qp->io_wq->pring;
@@ -2180,8 +2179,9 @@ lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
 				abts_scsi += qp->abts_scsi_io_bufs;
 				abts_nvme += qp->abts_nvme_io_bufs;
 			}
-			if (!vport || !vport->localport ||
-			    vport->phba->hba_flag & HBA_PCI_ERR)
+			if (!vport->localport ||
+			    test_bit(HBA_PCI_ERR, &vport->phba->bit_flags) ||
+			    vport->load_flag & FC_UNLOADING)
 				return;
 
 			lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
@@ -2541,8 +2541,7 @@ lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 		 * return values is ignored.  The upcall is a courtesy to the
 		 * transport.
 		 */
-		if (vport->load_flag & FC_UNLOADING ||
-		    unlikely(vport->phba->hba_flag & HBA_PCI_ERR))
+		if (vport->load_flag & FC_UNLOADING)
 			(void)nvme_fc_set_remoteport_devloss(remoteport, 0);
 
 		ret = nvme_fc_unregister_remoteport(remoteport);
drivers/scsi/lpfc/lpfc_sli.c +10 -5
index 20d40957a385..26f6a147b5ae 100644
@@ -2828,6 +2828,12 @@ __lpfc_sli_rpi_release(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 	ndlp->nlp_flag &= ~NLP_UNREG_INP;
 }
 
+void
+lpfc_sli_rpi_release(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
+{
+	__lpfc_sli_rpi_release(vport, ndlp);
+}
+
 /**
  * lpfc_sli_def_mbox_cmpl - Default mailbox completion handler
  * @phba: Pointer to HBA context object.
@@ -4624,11 +4630,6 @@ lpfc_sli_flush_io_rings(struct lpfc_hba *phba)
 	struct lpfc_iocbq *piocb, *next_iocb;
 
 	spin_lock_irq(&phba->hbalock);
-	if (phba->hba_flag & HBA_IOQ_FLUSH ||
-	    !phba->sli4_hba.hdwq) {
-		spin_unlock_irq(&phba->hbalock);
-		return;
-	}
 	/* Indicate the I/O queues are flushed */
 	phba->hba_flag |= HBA_IOQ_FLUSH;
 	spin_unlock_irq(&phba->hbalock);
@@ -10997,6 +10998,10 @@ lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number,
 	unsigned long iflags;
 	int rc;
 
+	/* If the PCI channel is in offline state, do not post iocbs. */
+	if (unlikely(pci_channel_offline(phba->pcidev)))
+		return IOCB_ERROR;
+
 	if (phba->sli_rev == LPFC_SLI_REV4) {
 		lpfc_sli_prep_wqe(phba, piocb);
 
Signature · 1 line
2.26.2
#2 lore raw

PATCH 2/4 lpfc: Fix unload hang after back to back PCI EEH faults

In reply to James Smart

When injecting eeh errors the port is getting hung up waiting on the node list to empty, message number 0233. The driver is stuck at this point and also can't unload. The driver makes transport remoteport delete calls which try to abort I/O's, but the eeh daemon has already called the driver to detach and the detachment has set the global FC_UNLOADING flag. There are several code paths that will avoid I/O cleanup if the FC_UNLOADING flag is set, resulting in transports waiting for I/O while the driver is waiting on transports to clean up.

Additionally, During study of the list, a locking issue was found in lpfc_sli_abort_iocb_ring that could corrupt the list.

To Fix:
A special case was added to the lpfc_cleanup routine to call
lpfc_sli_flush_rings if the driver is FC_UNLOADING and if the pci-slot is offline (e.g. eeh).
Changed the sli4 part of lpfc_sli_abort_iocb_ring to use the ring_lock.
Also added code to cancel the IOs if the pci-slot is offline.
Added checks and returns for the FC_UNLOADING and HBA_IOQ_FLUSH flags
to prevent trying to send an IO that we cannot handle.

---
 drivers/scsi/lpfc/lpfc_hbadisc.c |  1 +
 drivers/scsi/lpfc/lpfc_init.c    | 26 +++++++++++++++--
 drivers/scsi/lpfc/lpfc_nvme.c    | 16 ++++++++--
 drivers/scsi/lpfc/lpfc_sli.c     | 50 ++++++++++++++++++++++----------
 4 files changed, 72 insertions(+), 21 deletions(-)
drivers/scsi/lpfc/lpfc_hbadisc.c +1 -0
index 6983c70f2fc6..2b877dff5ed4 100644
@@ -5422,6 +5422,7 @@ lpfc_unreg_rpi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 				ndlp->nlp_flag &= ~NLP_UNREG_INP;
 				mempool_free(mbox, phba->mbox_mem_pool);
 				acc_plogi = 1;
+				lpfc_nlp_put(ndlp);
 			}
 		} else {
 			lpfc_printf_vlog(vport, KERN_INFO,
drivers/scsi/lpfc/lpfc_init.c +23 -3
index b8ab6dcbadc5..cf83bc0e27c0 100644
@@ -95,6 +95,7 @@ static void lpfc_sli4_oas_verify(struct lpfc_hba *phba);
 static uint16_t lpfc_find_cpu_handle(struct lpfc_hba *, uint16_t, int);
 static void lpfc_setup_bg(struct lpfc_hba *, struct Scsi_Host *);
 static int lpfc_sli4_cgn_parm_chg_evt(struct lpfc_hba *);
+static void lpfc_sli4_prep_dev_for_reset(struct lpfc_hba *phba);
 
 static struct scsi_transport_template *lpfc_transport_template = NULL;
 static struct scsi_transport_template *lpfc_vport_transport_template = NULL;
@@ -1985,6 +1986,7 @@ lpfc_handle_eratt_s4(struct lpfc_hba *phba)
 	if (pci_channel_offline(phba->pcidev)) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
 				"3166 pci channel is offline\n");
+		lpfc_sli_flush_io_rings(phba);
 		return;
 	}
 
@@ -2973,6 +2975,22 @@ lpfc_cleanup(struct lpfc_vport *vport)
 					NLP_EVT_DEVICE_RM);
 	}
 
+	/* This is a special case flush to return all
+	 * IOs before entering this loop. There are
+	 * two points in the code where a flush is
+	 * avoided if the FC_UNLOADING flag is set.
+	 * one is in the multipool destroy,
+	 * (this prevents a crash) and the other is
+	 * in the nvme abort handler, ( also prevents
+	 * a crash). Both of these exceptions are
+	 * cases where the slot is still accessible.
+	 * The flush here is only when the pci slot
+	 * is offline.
+	 */
+	if (vport->load_flag & FC_UNLOADING &&
+	    pci_channel_offline(phba->pcidev))
+		lpfc_sli_flush_io_rings(vport->phba);
+
 	/* At this point, ALL ndlp's should be gone
 	 * because of the previous NLP_EVT_DEVICE_RM.
 	 * Lets wait for this to happen, if needed.
@@ -2985,7 +3003,7 @@ lpfc_cleanup(struct lpfc_vport *vport)
 			list_for_each_entry_safe(ndlp, next_ndlp,
 						&vport->fc_nodes, nlp_listp) {
 				lpfc_printf_vlog(ndlp->vport, KERN_ERR,
-						 LOG_TRACE_EVENT,
+						 LOG_DISCOVERY,
 						 "0282 did:x%x ndlp:x%px "
 						 "refcnt:%d xflags x%x nflag x%x\n",
 						 ndlp->nlp_DID, (void *)ndlp,
@@ -13359,8 +13377,9 @@ lpfc_sli4_hba_unset(struct lpfc_hba *phba)
 	/* Abort all iocbs associated with the hba */
 	lpfc_sli_hba_iocb_abort(phba);
 
-	/* Wait for completion of device XRI exchange busy */
-	lpfc_sli4_xri_exchange_busy_wait(phba);
+	if (!pci_channel_offline(phba->pcidev))
+		/* Wait for completion of device XRI exchange busy */
+		lpfc_sli4_xri_exchange_busy_wait(phba);
 
 	/* per-phba callback de-registration for hotplug event */
 	if (phba->pport)
@@ -14264,6 +14283,7 @@ lpfc_sli_prep_dev_for_perm_failure(struct lpfc_hba *phba)
 			"2711 PCI channel permanent disable for failure\n");
 	/* Block all SCSI devices' I/Os on the host */
 	lpfc_scsi_dev_block(phba);
+	lpfc_sli4_prep_dev_for_reset(phba);
 
 	/* stop all timers */
 	lpfc_stop_hba_timers(phba);
drivers/scsi/lpfc/lpfc_nvme.c +13 -3
index e47205e0d3e2..8d26f207ebd2 100644
@@ -93,6 +93,11 @@ lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,
 
 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
 	vport = lport->vport;
+
+	if (!vport || vport->load_flag & FC_UNLOADING ||
+	    vport->phba->hba_flag & HBA_IOQ_FLUSH)
+		return -ENODEV;
+
 	qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL);
 	if (qhandle == NULL)
 		return -ENOMEM;
@@ -267,7 +272,8 @@ lpfc_nvme_handle_lsreq(struct lpfc_hba *phba,
 		return -EINVAL;
 
 	remoteport = lpfc_rport->remoteport;
-	if (!vport->localport)
+	if (!vport->localport ||
+	    vport->phba->hba_flag & HBA_IOQ_FLUSH)
 		return -EINVAL;
 
 	lport = vport->localport->private;
@@ -559,6 +565,8 @@ __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 				 ndlp->nlp_DID, ntype, nstate);
 		return -ENODEV;
 	}
+	if (vport->phba->hba_flag & HBA_IOQ_FLUSH)
+		return -ENODEV;
 
 	if (!vport->phba->sli4_hba.nvmels_wq)
 		return -ENOMEM;
@@ -662,7 +670,8 @@ lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
 		return -EINVAL;
 
 	vport = lport->vport;
-	if (vport->load_flag & FC_UNLOADING)
+	if (vport->load_flag & FC_UNLOADING ||
+	    vport->phba->hba_flag & HBA_IOQ_FLUSH)
 		return -ENODEV;
 
 	atomic_inc(&lport->fc4NvmeLsRequests);
@@ -1516,7 +1525,8 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 
 	phba = vport->phba;
 
-	if (unlikely(vport->load_flag & FC_UNLOADING)) {
+	if ((unlikely(vport->load_flag & FC_UNLOADING)) ||
+	    phba->hba_flag & HBA_IOQ_FLUSH) {
 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
 				 "6124 Fail IO, Driver unload\n");
 		atomic_inc(&lport->xmt_fcp_err);
drivers/scsi/lpfc/lpfc_sli.c +35 -15

77 lines not on this page · raw message

Signature · 1 line
2.26.2
#3 lore raw

PATCH 3/4 lpfc: Fix queue failures when recovering from PCI parity error

In reply to James Smart

When recovering from a pci-parity error the driver is failing to re-create queues, causing recovery to fail. Looking deeper, it was found tha the interrupt vector count allocated on the recovery is fewer than the vectors originally allocated. This disparity resulted in cpu map entries with stale information. When the driver tries to re-create the queues, it attempts to use the stale information which indicates an eq/interrupt vector that was no longer created.

Fix by clearng the cpup map array before enabling and requesting the irqs in the lpfc_sli_reset_slot_s4 routine.

---
 drivers/scsi/lpfc/lpfc_init.c | 2 ++
 1 file changed, 2 insertions(+)
drivers/scsi/lpfc/lpfc_init.c +2 -0
index cf83bc0e27c0..461d333b1b3a 100644
@@ -15236,6 +15236,8 @@ lpfc_io_slot_reset_s4(struct pci_dev *pdev)
 	psli->sli_flag &= ~LPFC_SLI_ACTIVE;
 	spin_unlock_irq(&phba->hbalock);
 
+	/* Init cpu_map array */
+	lpfc_cpu_map_array_init(phba);
 	/* Configure and enable interrupt */
 	intr_mode = lpfc_sli4_enable_intr(phba, phba->intr_mode);
 	if (intr_mode == LPFC_INTR_ERROR) {
Signature · 1 line
2.26.2
#4 lore raw

PATCH 4/4 lpfc: Update lpfc version to 14.2.0.1

In reply to James Smart

Update lpfc version to 14.2.0.1

---
 drivers/scsi/lpfc/lpfc_version.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/scsi/lpfc/lpfc_version.h +1 -1
index e52f37e5d896..a4d3259b8c52 100644
@@ -20,7 +20,7 @@
  * included with this package.                                     *
  *******************************************************************/
 
-#define LPFC_DRIVER_VERSION "14.2.0.0"
+#define LPFC_DRIVER_VERSION "14.2.0.1"
 #define LPFC_DRIVER_NAME		"lpfc"
 
 /* Used for SLI 2/3 */
Signature · 1 line
2.26.2
#6 lore raw
In reply to James Smart
Quoted 8 lines · James Smart

Update lpfc to revision 14.2.0.1

This patch set contains 3 fixes for errors detected during eeh error injection.

The patches were cut against Martin's 5.18/scsi-staging tree

[...]

Applied to 5.18/scsi-fixes, thanks!

[1/4] lpfc: Improve PCI EEH Error and Recovery Handling

      https://git.kernel.org/mkp/scsi/c/35ed9613d83f
[2/4] lpfc: Fix unload hang after back to back PCI EEH faults
      https://git.kernel.org/mkp/scsi/c/a4691038b407
[3/4] lpfc: Fix queue failures when recovering from PCI parity error
      https://git.kernel.org/mkp/scsi/c/df0101197c4d
[4/4] lpfc: Update lpfc version to 14.2.0.1
      https://git.kernel.org/mkp/scsi/c/4f3beb36b1e4
Signature · 1 line
Martin K. Petersen	Oracle Linux Engineering