Skip to contentSkip to conversation

mpi3mr: Enhancements for mpi3mr

PATCH Applied Ranjan Kumar 2 participants 6 messages Mar 20 – Apr 3

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover mpi3mr: Enhancements for mpi3mr

Enhancements for mpi3mr driver

Ranjan Kumar (3):
mpi3mr: Reset controller on invalid I/O completion
mpi3mr: Add queue-full tracking for operational request queues
mpi3mr: Add retry mechanism for IOC shutdown with timeout reset

 drivers/scsi/mpi3mr/mpi3mr.h    | 16 ++++++++++
 drivers/scsi/mpi3mr/mpi3mr_fw.c | 55 +++++++++++++++++++++++++++++----
 drivers/scsi/mpi3mr/mpi3mr_os.c | 11 +++++--
 3 files changed, 74 insertions(+), 8 deletions(-)
Signature · 1 line
2.47.3
#1 lore raw

PATCH 1/3 mpi3mr: Reset controller on invalid I/O completion

Operational replies without a valid scsi_cmnd indicate an invalid I/O completion and a potentially inconsistent controller state. Track this condition and allow the watchdog to trigger a soft reset to safely recover.

---
 drivers/scsi/mpi3mr/mpi3mr.h    |  3 +++
 drivers/scsi/mpi3mr/mpi3mr_fw.c |  7 +++++++
 drivers/scsi/mpi3mr/mpi3mr_os.c | 11 +++++++++--
 3 files changed, 19 insertions(+), 2 deletions(-)
drivers/scsi/mpi3mr/mpi3mr.h +3 -0
index 6e962092577d..da141c185eef 100644
@@ -323,6 +323,7 @@ enum mpi3mr_reset_reason {
 	MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT = 29,
 	MPI3MR_RESET_FROM_SAS_TRANSPORT_TIMEOUT = 30,
 	MPI3MR_RESET_FROM_TRIGGER = 31,
+	MPI3MR_RESET_FROM_INVALID_COMPLETION = 32,
 };
 
 #define MPI3MR_RESET_REASON_OSTYPE_LINUX	1
@@ -1183,6 +1184,7 @@ struct scmd_priv {
  * @num_tb_segs: Number of Segments in Trace buffer
  * @trace_buf_pool: DMA pool for Segmented trace buffer segments
  * @trace_buf: Trace buffer segments memory descriptor
+ * @invalid_io_comp: Invalid IO completion
  */
 struct mpi3mr_ioc {
 	struct list_head list;
@@ -1394,6 +1396,7 @@ struct mpi3mr_ioc {
 	u32 num_tb_segs;
 	struct dma_pool *trace_buf_pool;
 	struct segments *trace_buf;
+	u8 invalid_io_comp;
 
 };
 
drivers/scsi/mpi3mr/mpi3mr_fw.c +7 -0
index 04d4a2aea7d7..58360666fb78 100644
@@ -996,6 +996,7 @@ static const struct {
 	{ MPI3MR_RESET_FROM_FIRMWARE, "firmware asynchronous reset" },
 	{ MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT, "configuration request timeout"},
 	{ MPI3MR_RESET_FROM_SAS_TRANSPORT_TIMEOUT, "timeout of a SAS transport layer request" },
+	{ MPI3MR_RESET_FROM_INVALID_COMPLETION, "invalid cmd completion" },
 };
 
 /**
@@ -2880,6 +2881,11 @@ static void mpi3mr_watchdog_work(struct work_struct *work)
 		return;
 	}
 
+	if (mrioc->invalid_io_comp) {
+		mpi3mr_soft_reset_handler(mrioc, MPI3MR_RESET_FROM_INVALID_COMPLETION, 1);
+		return;
+	}
+
 	if (atomic_read(&mrioc->admin_pend_isr)) {
 		ioc_err(mrioc, "Unprocessed admin ISR instance found\n"
 				"flush admin replies\n");
@@ -5647,6 +5653,7 @@ int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc,
 	ssleep(MPI3MR_RESET_TOPOLOGY_SETTLE_TIME);
 
 out:
+	mrioc->invalid_io_comp = 0;
 	if (!retval) {
 		mrioc->diagsave_timeout = 0;
 		mrioc->reset_in_progress = 0;
drivers/scsi/mpi3mr/mpi3mr_os.c +9 -2
index 8e5abf620718..59d43f768ae6 100644
@@ -3462,8 +3462,15 @@ void mpi3mr_process_op_reply_desc(struct mpi3mr_ioc *mrioc,
 	}
 	scmd = mpi3mr_scmd_from_host_tag(mrioc, host_tag, qidx);
 	if (!scmd) {
-		panic("%s: Cannot Identify scmd for host_tag 0x%x\n",
-		    mrioc->name, host_tag);
+		ioc_err(mrioc, "Cannot Identify scmd for host_tag 0x%x", host_tag);
+		ioc_err(mrioc,
+		    "reply_desc_type(%d) host_tag(%d(0x%04x)): qid(%d): command issued to\n"
+		    "handle(0x%04x) returned with ioc_status(0x%04x), log_info(0x%08x),\n"
+		    "scsi_state(0x%02x), scsi_status(0x%02x), xfer_count(%d), resp_data(0x%08x)\n",
+		    reply_desc_type, host_tag, host_tag, qidx+1, dev_handle, ioc_status,
+		    ioc_loginfo, scsi_state, scsi_status,  xfer_count,
+		    resp_data);
+		mrioc->invalid_io_comp = 1;
 		goto out;
 	}
 	priv = scsi_cmd_priv(scmd);
Signature · 1 line
2.47.3
#2 lore raw

PATCH 2/3 mpi3mr: Add queue-full tracking for operational request queues

In reply to Ranjan Kumar

Track queue-full conditions on operational request queues in the driver. Record the last host tag returned to the SCSI mid-layer
and count I/Os affected by queue-full conditions.

---
 drivers/scsi/mpi3mr/mpi3mr.h    | 12 ++++++++++++
 drivers/scsi/mpi3mr/mpi3mr_fw.c | 16 ++++++++++++++++
 2 files changed, 28 insertions(+)
drivers/scsi/mpi3mr/mpi3mr.h +12 -0
index da141c185eef..631a48f7425d 100644
@@ -429,6 +429,14 @@ struct segments {
  * @q_segments: Segment descriptor pointer
  * @q_segment_list: Segment list base virtual address
  * @q_segment_list_dma: Segment list base DMA address
+ * @last_full_host_tag: Hosttag of last IO returned to SML
+ *			due to queue full
+ * @qfull_io_count: Number of IOs returned back to SML
+ *			due to queue full
+ * @qfull_instances: Total queue full occurrences.One occurrence
+ *			starts with queue full detection and ends
+ *			with queue full breaks.
+ *
  */
 struct op_req_qinfo {
 	u16 ci;
@@ -442,6 +450,10 @@ struct op_req_qinfo {
 	struct segments *q_segments;
 	void *q_segment_list;
 	dma_addr_t q_segment_list_dma;
+	u16 last_full_host_tag;
+	u64 qfull_io_count;
+	u32 qfull_instances;
+
 };
 
 /**
drivers/scsi/mpi3mr/mpi3mr_fw.c +16 -0
index 58360666fb78..da0d475db01e 100644
@@ -2362,6 +2362,9 @@ static int mpi3mr_create_op_req_q(struct mpi3mr_ioc *mrioc, u16 idx,
 	op_req_q->ci = 0;
 	op_req_q->pi = 0;
 	op_req_q->reply_qid = reply_qid;
+	op_req_q->last_full_host_tag =  MPI3MR_HOSTTAG_INVALID;
+	op_req_q->qfull_io_count =  0;
+	op_req_q->qfull_instances =  0;
 	spin_lock_init(&op_req_q->q_lock);
 
 	if (!op_req_q->q_segments) {
@@ -2549,6 +2552,8 @@ int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc,
 	u16 req_sz = mrioc->facts.op_req_sz;
 	struct segments *segments = op_req_q->q_segments;
 	struct op_reply_qinfo *op_reply_q = NULL;
+	struct mpi3_scsi_io_request *scsiio_req =
+		(struct mpi3_scsi_io_request *)req;
 
 	reply_qidx = op_req_q->reply_qid - 1;
 	op_reply_q = mrioc->op_reply_qinfo + reply_qidx;
@@ -2566,11 +2571,21 @@ int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc,
 		mpi3mr_process_op_reply_q(mrioc, mrioc->intr_info[midx].op_reply_q);
 
 		if (mpi3mr_check_req_qfull(op_req_q)) {
+
+			if (op_req_q->last_full_host_tag ==
+			    MPI3MR_HOSTTAG_INVALID)
+				op_req_q->qfull_instances++;
+
+			op_req_q->last_full_host_tag = scsiio_req->host_tag;
+			op_req_q->qfull_io_count++;
 			retval = -EAGAIN;
 			goto out;
 		}
 	}
 
+	if (op_req_q->last_full_host_tag != MPI3MR_HOSTTAG_INVALID)
+		op_req_q->last_full_host_tag = MPI3MR_HOSTTAG_INVALID;
+
 	if (mrioc->reset_in_progress) {
 		ioc_err(mrioc, "OpReqQ submit reset in progress\n");
 		retval = -EAGAIN;
@@ -4829,6 +4844,7 @@ void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc)
 			mrioc->req_qinfo[i].qid = 0;
 			mrioc->req_qinfo[i].reply_qid = 0;
 			spin_lock_init(&mrioc->req_qinfo[i].q_lock);
+			mrioc->req_qinfo[i].last_full_host_tag = 0;
 			mpi3mr_memset_op_req_q_buffers(mrioc, i);
 		}
 	}
Signature · 1 line
2.47.3
#3 lore raw

PATCH 3/3 mpi3mr: Add retry mechanism for IOC shutdown with timeout reset

In reply to Ranjan Kumar

Enhance the IOC shutdown process to handle transient failures during controller cleanup. Add retry logic with configurable maximum retry count (MPI3MR_MAX_SHUTDOWN_RETRY_COUNT) and proper timeout management that resets on each retry attempt. This ensures shutdown can recover from temporary issues without failing completely.

---
 drivers/scsi/mpi3mr/mpi3mr.h    |  1 +
 drivers/scsi/mpi3mr/mpi3mr_fw.c | 32 ++++++++++++++++++++++++++------
 2 files changed, 27 insertions(+), 6 deletions(-)
drivers/scsi/mpi3mr/mpi3mr.h +1 -0
index 631a48f7425d..c25525fe0671 100644
@@ -159,6 +159,7 @@ extern atomic64_t event_counter;
 /* Controller Reset related definitions */
 #define MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT	5
 #define MPI3MR_MAX_RESET_RETRY_COUNT		3
+#define MPI3MR_MAX_SHUTDOWN_RETRY_COUNT		2
 
 /* ResponseCode definitions */
 #define MPI3MR_RI_MASK_RESPCODE		(0x000000FF)
drivers/scsi/mpi3mr/mpi3mr_fw.c +26 -6
index da0d475db01e..7983c3c01c65 100644
@@ -5061,9 +5061,9 @@ void mpi3mr_free_mem(struct mpi3mr_ioc *mrioc)
  */
 static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc)
 {
-	u32 ioc_config, ioc_status;
-	u8 retval = 1;
-	u32 timeout = MPI3MR_DEFAULT_SHUTDOWN_TIME * 10;
+	u32 ioc_config, ioc_status, shutdown_action;
+	u8 retval = 1, retry = 0;
+	u32 timeout = MPI3MR_DEFAULT_SHUTDOWN_TIME * 10, timeout_remaining = 0;
 
 	ioc_info(mrioc, "Issuing shutdown Notification\n");
 	if (mrioc->unrecoverable) {
@@ -5078,14 +5078,16 @@ static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc)
 		return;
 	}
 
+	shutdown_action = MPI3_SYSIF_IOC_CONFIG_SHUTDOWN_NORMAL |
+	    MPI3_SYSIF_IOC_CONFIG_DEVICE_SHUTDOWN_SEND_REQ;
 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
-	ioc_config |= MPI3_SYSIF_IOC_CONFIG_SHUTDOWN_NORMAL;
-	ioc_config |= MPI3_SYSIF_IOC_CONFIG_DEVICE_SHUTDOWN_SEND_REQ;
+	ioc_config |= shutdown_action;
 
 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
 
 	if (mrioc->facts.shutdown_timeout)
 		timeout = mrioc->facts.shutdown_timeout * 10;
+	timeout_remaining = timeout;
 
 	do {
 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);
@@ -5094,8 +5096,26 @@ static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc)
 			retval = 0;
 			break;
 		}
+		if (mrioc->unrecoverable)
+			break;
+		if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) {
+			mpi3mr_print_fault_info(mrioc);
+			if (retry >= MPI3MR_MAX_SHUTDOWN_RETRY_COUNT)
+				break;
+			if (mpi3mr_issue_reset(mrioc,
+			    MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
+			    MPI3MR_RESET_FROM_CTLR_CLEANUP))
+				break;
+			ioc_config =
+			    readl(&mrioc->sysif_regs->ioc_configuration);
+			ioc_config |= shutdown_action;
+			writel(ioc_config,
+			    &mrioc->sysif_regs->ioc_configuration);
+			timeout_remaining = timeout;
+			retry++;
+		}
 		msleep(100);
-	} while (--timeout);
+	} while (--timeout_remaining);
 
 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
Signature · 1 line
2.47.3
#5 lore raw
In reply to Ranjan Kumar
Quoted 8 lines · Ranjan Kumar

Enhancements for mpi3mr driver

Ranjan Kumar (3):
mpi3mr: Reset controller on invalid I/O completion
mpi3mr: Add queue-full tracking for operational request queues
mpi3mr: Add retry mechanism for IOC shutdown with timeout reset

[...]

Applied to 7.1/scsi-queue, thanks!

[1/3] mpi3mr: Reset controller on invalid I/O completion

      https://git.kernel.org/mkp/scsi/c/31693fbbfa21
[2/3] mpi3mr: Add queue-full tracking for operational request queues
      https://git.kernel.org/mkp/scsi/c/9d660e482071
[3/3] mpi3mr: Add retry mechanism for IOC shutdown with timeout reset
      https://git.kernel.org/mkp/scsi/c/02ff1d2bcf2d
Signature · 1 line
Martin K. Petersen