Skip to contentSkip to conversation

UFS bug fixes

PATCH Superseded Jaegeuk Kim 2 participants 11 messages Jan 7, 2021 SCSI

v4 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

v4 cover UFS bug fixes

Change log from v3:
- move ufshcd_clear_ua_wluns() after ufshcd_scsi_add_wlus()
- remove BLK_MQ_REQ_RESERVED for tm tag
- move IO wait to cover all the non-fatal errors

#1 lore raw

PATCH v4 1/2 scsi: ufs: fix livelock of ufshcd_clear_ua_wluns

When gate_work/ungate_work gets an error during hibern8_enter or exit,
ufshcd_err_handler()
ufshcd_scsi_block_requests()
ufshcd_reset_and_restore()

     ufshcd_clear_ua_wluns() -> stuck

ufshcd_scsi_unblock_requests()

In order to avoid it, ufshcd_clear_ua_wluns() can be called per recovery flows such as suspend/resume, link_recovery, and error_handler.

Fixes 1918651f2d7e ("scsi: ufs: Clear UAC for RPMB after ufshcd resets")
---
 drivers/scsi/ufs/ufshcd.c | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)
drivers/scsi/ufs/ufshcd.c +12 -7
index bedb822a40a3..e6e7bdf99cd7 100644
@@ -3996,6 +3996,8 @@ int ufshcd_link_recovery(struct ufs_hba *hba)
 	if (ret)
 		dev_err(hba->dev, "%s: link recovery failed, err %d",
 			__func__, ret);
+	else
+		ufshcd_clear_ua_wluns(hba);
 
 	return ret;
 }
@@ -6003,6 +6005,9 @@ static void ufshcd_err_handler(struct work_struct *work)
 	ufshcd_scsi_unblock_requests(hba);
 	ufshcd_err_handling_unprepare(hba);
 	up(&hba->eh_sem);
+
+	if (!err && needs_reset)
+		ufshcd_clear_ua_wluns(hba);
 }
 
 /**
@@ -6940,14 +6945,11 @@ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
 	ufshcd_set_clk_freq(hba, true);
 
 	err = ufshcd_hba_enable(hba);
-	if (err)
-		goto out;
 
 	/* Establish the link again and restore the device */
-	err = ufshcd_probe_hba(hba, false);
 	if (!err)
-		ufshcd_clear_ua_wluns(hba);
-out:
+		err = ufshcd_probe_hba(hba, false);
+
 	if (err)
 		dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
 	ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err);
@@ -7718,6 +7720,8 @@ static int ufshcd_add_lus(struct ufs_hba *hba)
 	if (ret)
 		goto out;
 
+	ufshcd_clear_ua_wluns(hba);
+
 	/* Initialize devfreq after UFS device is detected */
 	if (ufshcd_is_clkscaling_supported(hba)) {
 		memcpy(&hba->clk_scaling.saved_pwr_info.info,
@@ -7919,8 +7923,6 @@ static void ufshcd_async_scan(void *data, async_cookie_t cookie)
 		pm_runtime_put_sync(hba->dev);
 		ufshcd_exit_clk_scaling(hba);
 		ufshcd_hba_exit(hba);
-	} else {
-		ufshcd_clear_ua_wluns(hba);
 	}
 }
 
@@ -8777,6 +8779,7 @@ static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
 		ufshcd_resume_clkscaling(hba);
 	hba->clk_gating.is_suspended = false;
 	hba->dev_info.b_rpm_dev_flush_capable = false;
+	ufshcd_clear_ua_wluns(hba);
 	ufshcd_release(hba);
 out:
 	if (hba->dev_info.b_rpm_dev_flush_capable) {
@@ -8887,6 +8890,8 @@ static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
 		cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
 	}
 
+	ufshcd_clear_ua_wluns(hba);
+
 	/* Schedule clock gating in case of no access to UFS device yet */
 	ufshcd_release(hba);
 
Signature · 1 line
2.29.2.729.g45daf8777d-goog
Reviewed-by on v4 1/2
#2 lore raw

Re: [PATCH v4 1/2] scsi: ufs: fix livelock of ufshcd_clear_ua_wluns

Quoted 87 lines · Jaegeuk Kim

When gate_work/ungate_work gets an error during hibern8_enter or exit,
ufshcd_err_handler()
ufshcd_scsi_block_requests()
ufshcd_reset_and_restore()

     ufshcd_clear_ua_wluns() -> stuck

ufshcd_scsi_unblock_requests()

In order to avoid it, ufshcd_clear_ua_wluns() can be called per recovery flows such as suspend/resume, link_recovery, and error_handler.

Fixes 1918651f2d7e ("scsi: ufs: Clear UAC for RPMB after ufshcd resets")
---
 drivers/scsi/ufs/ufshcd.c | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)
drivers/scsi/ufs/ufshcd.c +5 -0
index bedb822a40a3..e6e7bdf99cd7 100644
@@ -3996,6 +3996,8 @@ int ufshcd_link_recovery(struct ufs_hba *hba)
 	if (ret)
 		dev_err(hba->dev, "%s: link recovery failed, err %d",
 			__func__, ret);
+	else
+		ufshcd_clear_ua_wluns(hba);
 
 	return ret;
 }
@@ -6003,6 +6005,9 @@ static void ufshcd_err_handler(struct work_struct 
 *work)
 	ufshcd_scsi_unblock_requests(hba);
 	ufshcd_err_handling_unprepare(hba);
 	up(&hba->eh_sem);
+
+	if (!err && needs_reset)
+		ufshcd_clear_ua_wluns(hba);
 }
 

/**

drivers/scsi/ufs/ufshcd.c +2 -5
@@ -6940,14 +6945,11 @@ static int
 ufshcd_host_reset_and_restore(struct ufs_hba *hba)
 	ufshcd_set_clk_freq(hba, true);
 
 	err = ufshcd_hba_enable(hba);
-	if (err)
-		goto out;
 
 	/* Establish the link again and restore the device */
-	err = ufshcd_probe_hba(hba, false);
 	if (!err)
-		ufshcd_clear_ua_wluns(hba);
-out:
+		err = ufshcd_probe_hba(hba, false);
+
 	if (err)
 		dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
drivers/scsi/ufs/ufshcd.c +3 -2
 	ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err);
@@ -7718,6 +7720,8 @@ static int ufshcd_add_lus(struct ufs_hba *hba)
 	if (ret)
 		goto out;
 
+	ufshcd_clear_ua_wluns(hba);
+
 	/* Initialize devfreq after UFS device is detected */
 	if (ufshcd_is_clkscaling_supported(hba)) {
 		memcpy(&hba->clk_scaling.saved_pwr_info.info,
@@ -7919,8 +7923,6 @@ static void ufshcd_async_scan(void *data,
 async_cookie_t cookie)
 		pm_runtime_put_sync(hba->dev);
 		ufshcd_exit_clk_scaling(hba);
 		ufshcd_hba_exit(hba);
-	} else {
-		ufshcd_clear_ua_wluns(hba);
 	}
 }
@@ -8777,6 +8779,7 @@ static int ufshcd_suspend(struct ufs_hba *hba,
 enum ufs_pm_op pm_op)
 		ufshcd_resume_clkscaling(hba);
 	hba->clk_gating.is_suspended = false;
 	hba->dev_info.b_rpm_dev_flush_capable = false;
+	ufshcd_clear_ua_wluns(hba);
 	ufshcd_release(hba);
 out:
drivers/scsi/ufs/ufshcd.c +0 -0
 	if (hba->dev_info.b_rpm_dev_flush_capable) {

drivers/scsi/ufs/ufshcd.c static int ufshcd_resume(struct ufs_hba *hba, · line 8897

Quoted 8 lines
drivers/scsi/ufs/ufshcd.c +2 -0
@@ -8887,6 +8890,8 @@ static int ufshcd_resume(struct ufs_hba *hba,
 enum ufs_pm_op pm_op)
 		cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
 	}
 
+	ufshcd_clear_ua_wluns(hba);
+
 	/* Schedule clock gating in case of no access to UFS device yet */
 	ufshcd_release(hba);
#3 lore raw

PATCH v4 2/2 scsi: ufs: handle LINERESET with correct tm_cmd

In reply to Jaegeuk Kim

From: Jaegeuk Kim <[email protected]>

This fixes a warning caused by wrong reserve tag usage in __ufshcd_issue_tm_cmd.

WARNING: CPU: 7 PID: 7 at block/blk-core.c:630 blk_get_request+0x68/0x70
WARNING: CPU: 4 PID: 157 at block/blk-mq-tag.c:82 blk_mq_get_tag+0x438/0x46c

And, in ufshcd_err_handler(), we can avoid to send tm_cmd before aborting outstanding commands by waiting a bit for IO completion like this.

__ufshcd_issue_tm_cmd: task management cmd 0x80 timed-out

Fixes 69a6c269c097 ("scsi: ufs: Use blk_{get,put}_request() to allocate and free TMFs")
Fixes 2355b66ed20c ("scsi: ufs: Handle LINERESET indication in err handler")
---
 drivers/scsi/ufs/ufshcd.c | 35 +++++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)
drivers/scsi/ufs/ufshcd.c +31 -4
index e6e7bdf99cd7..340dd5e515dd 100644
@@ -44,6 +44,9 @@
 /* Query request timeout */
 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
 
+/* LINERESET TIME OUT */
+#define LINERESET_IO_TIMEOUT_MS			(30000) /* 30 sec */
+
 /* Task management command timeout */
 #define TM_CMD_TIMEOUT	100 /* msecs */
 
@@ -5826,6 +5829,7 @@ static void ufshcd_err_handler(struct work_struct *work)
 	int err = 0, pmc_err;
 	int tag;
 	bool needs_reset = false, needs_restore = false;
+	ktime_t start;
 
 	hba = container_of(work, struct ufs_hba, eh_work);
 
@@ -5911,6 +5915,22 @@ static void ufshcd_err_handler(struct work_struct *work)
 	}
 
 	hba->silence_err_logs = true;
+
+	/* Wait for IO completion for non-fatal errors to avoid aborting IOs */
+	start = ktime_get();
+	while (hba->outstanding_reqs) {
+		ufshcd_complete_requests(hba);
+		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		schedule();
+		spin_lock_irqsave(hba->host->host_lock, flags);
+		if (ktime_to_ms(ktime_sub(ktime_get(), start)) >
+						LINERESET_IO_TIMEOUT_MS) {
+			dev_err(hba->dev, "%s: timeout, outstanding=0x%lx\n",
+					__func__, hba->outstanding_reqs);
+			break;
+		}
+	}
+
 	/* release lock as clear command might sleep */
 	spin_unlock_irqrestore(hba->host->host_lock, flags);
 	/* Clear pending transfer requests */
@@ -6302,9 +6322,13 @@ static irqreturn_t ufshcd_intr(int irq, void *__hba)
 		intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 	}
 
-	if (enabled_intr_status && retval == IRQ_NONE) {
-		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x\n",
-					__func__, intr_status);
+	if (enabled_intr_status && retval == IRQ_NONE &&
+				!ufshcd_eh_in_progress(hba)) {
+		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n",
+					__func__,
+					intr_status,
+					hba->ufs_stats.last_intr_status,
+					enabled_intr_status);
 		ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
 	}
 
@@ -6348,7 +6372,10 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
 	 * Even though we use wait_event() which sleeps indefinitely,
 	 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
 	 */
-	req = blk_get_request(q, REQ_OP_DRV_OUT, BLK_MQ_REQ_RESERVED);
+	req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
 	req->end_io_data = &wait;
 	free_slot = req->tag;
 	WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);
Signature · 1 line
2.29.2.729.g45daf8777d-goog
#4 lore raw

Re: [PATCH v4 2/2] scsi: ufs: handle LINERESET with correct tm_cmd

Quoted 7 lines · Jaegeuk Kim

From: Jaegeuk Kim <[email protected]>

This fixes a warning caused by wrong reserve tag usage in __ufshcd_issue_tm_cmd.

WARNING: CPU: 7 PID: 7 at block/blk-core.c:630 blk_get_request+0x68/0x70
WARNING: CPU: 4 PID: 157 at block/blk-mq-tag.c:82 blk_mq_get_tag+0x438/0x46c
Quoted 8 lines

And, in ufshcd_err_handler(), we can avoid to send tm_cmd before aborting outstanding commands by waiting a bit for IO completion like this.

__ufshcd_issue_tm_cmd: task management cmd 0x80 timed-out

Fixes 69a6c269c097 ("scsi: ufs: Use blk_{get,put}_request() to

allocate and free TMFs")

Fixes 2355b66ed20c ("scsi: ufs: Handle LINERESET indication in err handler")

Hi Jaegeuk,

Sorry, what is wrong with commit 2355b66ed20c? Clearing pending I/O reqs is a general procedure for handling all non-fatal errors.

Thanks,
Can Guo.

Quoted 85 lines
---
 drivers/scsi/ufs/ufshcd.c | 35 +++++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)
drivers/scsi/ufs/ufshcd.c +20 -0
index e6e7bdf99cd7..340dd5e515dd 100644
@@ -44,6 +44,9 @@
 /* Query request timeout */
 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
 
+/* LINERESET TIME OUT */
+#define LINERESET_IO_TIMEOUT_MS			(30000) /* 30 sec */
+
 /* Task management command timeout */
 #define TM_CMD_TIMEOUT	100 /* msecs */
 
@@ -5826,6 +5829,7 @@ static void ufshcd_err_handler(struct work_struct 
 *work)
 	int err = 0, pmc_err;
 	int tag;
 	bool needs_reset = false, needs_restore = false;
+	ktime_t start;
 
 	hba = container_of(work, struct ufs_hba, eh_work);
@@ -5911,6 +5915,22 @@ static void ufshcd_err_handler(struct 
 work_struct *work)
 	}
 
 	hba->silence_err_logs = true;
+
+	/* Wait for IO completion for non-fatal errors to avoid aborting IOs 
 */
+	start = ktime_get();
+	while (hba->outstanding_reqs) {
+		ufshcd_complete_requests(hba);
+		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		schedule();
+		spin_lock_irqsave(hba->host->host_lock, flags);
+		if (ktime_to_ms(ktime_sub(ktime_get(), start)) >
+						LINERESET_IO_TIMEOUT_MS) {
+			dev_err(hba->dev, "%s: timeout, outstanding=0x%lx\n",
+					__func__, hba->outstanding_reqs);
+			break;
+		}
+	}
+
 	/* release lock as clear command might sleep */
drivers/scsi/ufs/ufshcd.c +7 -3
 	spin_unlock_irqrestore(hba->host->host_lock, flags);
 	/* Clear pending transfer requests */
@@ -6302,9 +6322,13 @@ static irqreturn_t ufshcd_intr(int irq, void 
 *__hba)
 		intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 	}
 
-	if (enabled_intr_status && retval == IRQ_NONE) {
-		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x\n",
-					__func__, intr_status);
+	if (enabled_intr_status && retval == IRQ_NONE &&
+				!ufshcd_eh_in_progress(hba)) {
+		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 
 0x%08x)\n",
+					__func__,
+					intr_status,
+					hba->ufs_stats.last_intr_status,
+					enabled_intr_status);
 		ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
drivers/scsi/ufs/ufshcd.c +4 -1
 	}
@@ -6348,7 +6372,10 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba 
 *hba,
 	 * Even though we use wait_event() which sleeps indefinitely,
 	 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
 	 */
-	req = blk_get_request(q, REQ_OP_DRV_OUT, BLK_MQ_REQ_RESERVED);
+	req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
 	req->end_io_data = &wait;
 	free_slot = req->tag;

WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);

#5 lore raw

Re: [PATCH v4 2/2] scsi: ufs: handle LINERESET with correct tm_cmd

Quoted 21 lines · Can Guo
Quoted 13 lines · Jaegeuk Kim

From: Jaegeuk Kim <[email protected]>

This fixes a warning caused by wrong reserve tag usage in __ufshcd_issue_tm_cmd.

WARNING: CPU: 7 PID: 7 at block/blk-core.c:630 blk_get_request+0x68/0x70
WARNING: CPU: 4 PID: 157 at block/blk-mq-tag.c:82
blk_mq_get_tag+0x438/0x46c

And, in ufshcd_err_handler(), we can avoid to send tm_cmd before aborting
outstanding commands by waiting a bit for IO completion like this.

Quoted 6 lines

__ufshcd_issue_tm_cmd: task management cmd 0x80 timed-out

Fixes 69a6c269c097 ("scsi: ufs: Use blk_{get,put}_request() to

allocate and free TMFs")

Fixes 2355b66ed20c ("scsi: ufs: Handle LINERESET indication in err

handler")

Quoted 4 lines

Hi Jaegeuk,

Sorry, what is wrong with commit 2355b66ed20c? Clearing pending I/O reqs is a general procedure for handling all non-fatal errors.

Without waiting IOs, I hit the below timeout all the time from LINERESET, which causes UFS stuck permanently, as mentioned in the description.

"__ufshcd_issue_tm_cmd: task management cmd 0x80 timed-out"

Quoted 89 lines

Thanks,
Can Guo.

Quoted 85 lines
---
 drivers/scsi/ufs/ufshcd.c | 35 +++++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)
drivers/scsi/ufs/ufshcd.c +20 -0
index e6e7bdf99cd7..340dd5e515dd 100644
@@ -44,6 +44,9 @@
 /* Query request timeout */
 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
 
+/* LINERESET TIME OUT */
+#define LINERESET_IO_TIMEOUT_MS			(30000) /* 30 sec */
+
 /* Task management command timeout */
 #define TM_CMD_TIMEOUT	100 /* msecs */
 
@@ -5826,6 +5829,7 @@ static void ufshcd_err_handler(struct work_struct
 *work)
 	int err = 0, pmc_err;
 	int tag;
 	bool needs_reset = false, needs_restore = false;
+	ktime_t start;
 
 	hba = container_of(work, struct ufs_hba, eh_work);
@@ -5911,6 +5915,22 @@ static void ufshcd_err_handler(struct work_struct
 *work)
 	}
 
 	hba->silence_err_logs = true;
+
+	/* Wait for IO completion for non-fatal errors to avoid aborting IOs
 */
+	start = ktime_get();
+	while (hba->outstanding_reqs) {
+		ufshcd_complete_requests(hba);
+		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		schedule();
+		spin_lock_irqsave(hba->host->host_lock, flags);
+		if (ktime_to_ms(ktime_sub(ktime_get(), start)) >
+						LINERESET_IO_TIMEOUT_MS) {
+			dev_err(hba->dev, "%s: timeout, outstanding=0x%lx\n",
+					__func__, hba->outstanding_reqs);
+			break;
+		}
+	}
+
 	/* release lock as clear command might sleep */
drivers/scsi/ufs/ufshcd.c +7 -3
 	spin_unlock_irqrestore(hba->host->host_lock, flags);
 	/* Clear pending transfer requests */
@@ -6302,9 +6322,13 @@ static irqreturn_t ufshcd_intr(int irq, void
 *__hba)
 		intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 	}
 
-	if (enabled_intr_status && retval == IRQ_NONE) {
-		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x\n",
-					__func__, intr_status);
+	if (enabled_intr_status && retval == IRQ_NONE &&
+				!ufshcd_eh_in_progress(hba)) {
+		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x,
 0x%08x)\n",
+					__func__,
+					intr_status,
+					hba->ufs_stats.last_intr_status,
+					enabled_intr_status);
 		ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
drivers/scsi/ufs/ufshcd.c +4 -1
 	}
@@ -6348,7 +6372,10 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba
 *hba,
 	 * Even though we use wait_event() which sleeps indefinitely,
 	 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
 	 */
-	req = blk_get_request(q, REQ_OP_DRV_OUT, BLK_MQ_REQ_RESERVED);
+	req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
 	req->end_io_data = &wait;
 	free_slot = req->tag;

WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);

#6 lore raw

Re: [PATCH v4 2/2] scsi: ufs: handle LINERESET with correct tm_cmd

Quoted 24 lines · Jaegeuk Kim
Quoted 23 lines · Can Guo
Quoted 13 lines · Jaegeuk Kim

From: Jaegeuk Kim <[email protected]>

This fixes a warning caused by wrong reserve tag usage in __ufshcd_issue_tm_cmd.

WARNING: CPU: 7 PID: 7 at block/blk-core.c:630 blk_get_request+0x68/0x70
WARNING: CPU: 4 PID: 157 at block/blk-mq-tag.c:82
blk_mq_get_tag+0x438/0x46c

And, in ufshcd_err_handler(), we can avoid to send tm_cmd before aborting
outstanding commands by waiting a bit for IO completion like this.

Quoted 6 lines

__ufshcd_issue_tm_cmd: task management cmd 0x80 timed-out

Fixes 69a6c269c097 ("scsi: ufs: Use blk_{get,put}_request() to

allocate and free TMFs")

Fixes 2355b66ed20c ("scsi: ufs: Handle LINERESET indication in err

handler")

Hi Jaegeuk,

Quoted 7 lines
Quoted 2 lines

Sorry, what is wrong with commit 2355b66ed20c? Clearing pending I/O reqs is a general procedure for handling all non-fatal errors.

Without waiting IOs, I hit the below timeout all the time from LINERESET, which causes UFS stuck permanently, as mentioned in the description.

"__ufshcd_issue_tm_cmd: task management cmd 0x80 timed-out"

In that case, ufshcd_try_to_abort_task(), the caller of __ufshcd_issue_tm_cmd(), should return -ETIMEOUT, then err_handler would jump to do a full reset, then bail. I am not sure what gets UFS stuck permanently. Could you please share the callstack if possible? I really want to know what is happening. Thanks.

Regards,
Can Guo.

Quoted 90 lines
Quoted 90 lines

Thanks,
Can Guo.

Quoted 85 lines
---
 drivers/scsi/ufs/ufshcd.c | 35 +++++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)
drivers/scsi/ufs/ufshcd.c +20 -0
index e6e7bdf99cd7..340dd5e515dd 100644
@@ -44,6 +44,9 @@
 /* Query request timeout */
 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
 
+/* LINERESET TIME OUT */
+#define LINERESET_IO_TIMEOUT_MS			(30000) /* 30 sec */
+
 /* Task management command timeout */
 #define TM_CMD_TIMEOUT	100 /* msecs */
 
@@ -5826,6 +5829,7 @@ static void ufshcd_err_handler(struct work_struct
 *work)
 	int err = 0, pmc_err;
 	int tag;
 	bool needs_reset = false, needs_restore = false;
+	ktime_t start;
 
 	hba = container_of(work, struct ufs_hba, eh_work);
@@ -5911,6 +5915,22 @@ static void ufshcd_err_handler(struct work_struct
 *work)
 	}
 
 	hba->silence_err_logs = true;
+
+	/* Wait for IO completion for non-fatal errors to avoid aborting IOs
 */
+	start = ktime_get();
+	while (hba->outstanding_reqs) {
+		ufshcd_complete_requests(hba);
+		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		schedule();
+		spin_lock_irqsave(hba->host->host_lock, flags);
+		if (ktime_to_ms(ktime_sub(ktime_get(), start)) >
+						LINERESET_IO_TIMEOUT_MS) {
+			dev_err(hba->dev, "%s: timeout, outstanding=0x%lx\n",
+					__func__, hba->outstanding_reqs);
+			break;
+		}
+	}
+
 	/* release lock as clear command might sleep */
drivers/scsi/ufs/ufshcd.c +7 -3
 	spin_unlock_irqrestore(hba->host->host_lock, flags);
 	/* Clear pending transfer requests */
@@ -6302,9 +6322,13 @@ static irqreturn_t ufshcd_intr(int irq, void
 *__hba)
 		intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 	}
 
-	if (enabled_intr_status && retval == IRQ_NONE) {
-		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x\n",
-					__func__, intr_status);
+	if (enabled_intr_status && retval == IRQ_NONE &&
+				!ufshcd_eh_in_progress(hba)) {
+		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x,
 0x%08x)\n",
+					__func__,
+					intr_status,
+					hba->ufs_stats.last_intr_status,
+					enabled_intr_status);
 		ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
drivers/scsi/ufs/ufshcd.c +4 -1
 	}
@@ -6348,7 +6372,10 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba
 *hba,
 	 * Even though we use wait_event() which sleeps indefinitely,
 	 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
 	 */
-	req = blk_get_request(q, REQ_OP_DRV_OUT, BLK_MQ_REQ_RESERVED);
+	req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
 	req->end_io_data = &wait;
 	free_slot = req->tag;

WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);

#7 lore raw

Re: [PATCH v4 2/2] scsi: ufs: handle LINERESET with correct tm_cmd

Quoted 34 lines · Can Guo
Quoted 24 lines · Jaegeuk Kim
Quoted 23 lines · Can Guo
Quoted 13 lines · Jaegeuk Kim

From: Jaegeuk Kim <[email protected]>

This fixes a warning caused by wrong reserve tag usage in __ufshcd_issue_tm_cmd.

WARNING: CPU: 7 PID: 7 at block/blk-core.c:630 blk_get_request+0x68/0x70
WARNING: CPU: 4 PID: 157 at block/blk-mq-tag.c:82
blk_mq_get_tag+0x438/0x46c

And, in ufshcd_err_handler(), we can avoid to send tm_cmd before aborting
outstanding commands by waiting a bit for IO completion like this.

Quoted 6 lines

__ufshcd_issue_tm_cmd: task management cmd 0x80 timed-out

Fixes 69a6c269c097 ("scsi: ufs: Use blk_{get,put}_request() to

allocate and free TMFs")

Fixes 2355b66ed20c ("scsi: ufs: Handle LINERESET indication in err

handler")

Hi Jaegeuk,

Quoted 8 lines
Quoted 2 lines

Sorry, what is wrong with commit 2355b66ed20c? Clearing pending I/O reqs is a general procedure for handling all non-fatal errors.

Without waiting IOs, I hit the below timeout all the time from LINERESET, which
causes UFS stuck permanently, as mentioned in the description.

"__ufshcd_issue_tm_cmd: task management cmd 0x80 timed-out"

Quoted 7 lines

In that case, ufshcd_try_to_abort_task(), the caller of __ufshcd_issue_tm_cmd(),
should return -ETIMEOUT, then err_handler would jump to do a full reset, then bail.
I am not sure what gets UFS stuck permanently. Could you please share the callstack
if possible? I really want to know what is happening. Thanks.

I can't share all the log tho, it entered full reset. While printing out whole registers, the device was hard reset. Thanks,

Quoted 94 lines

Regards,
Can Guo.

Quoted 90 lines
Quoted 90 lines

Thanks,
Can Guo.

Quoted 85 lines
---
 drivers/scsi/ufs/ufshcd.c | 35 +++++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)
drivers/scsi/ufs/ufshcd.c +20 -0
index e6e7bdf99cd7..340dd5e515dd 100644
@@ -44,6 +44,9 @@
 /* Query request timeout */
 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
 
+/* LINERESET TIME OUT */
+#define LINERESET_IO_TIMEOUT_MS			(30000) /* 30 sec */
+
 /* Task management command timeout */
 #define TM_CMD_TIMEOUT	100 /* msecs */
 
@@ -5826,6 +5829,7 @@ static void ufshcd_err_handler(struct work_struct
 *work)
 	int err = 0, pmc_err;
 	int tag;
 	bool needs_reset = false, needs_restore = false;
+	ktime_t start;
 
 	hba = container_of(work, struct ufs_hba, eh_work);
@@ -5911,6 +5915,22 @@ static void ufshcd_err_handler(struct work_struct
 *work)
 	}
 
 	hba->silence_err_logs = true;
+
+	/* Wait for IO completion for non-fatal errors to avoid aborting IOs
 */
+	start = ktime_get();
+	while (hba->outstanding_reqs) {
+		ufshcd_complete_requests(hba);
+		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		schedule();
+		spin_lock_irqsave(hba->host->host_lock, flags);
+		if (ktime_to_ms(ktime_sub(ktime_get(), start)) >
+						LINERESET_IO_TIMEOUT_MS) {
+			dev_err(hba->dev, "%s: timeout, outstanding=0x%lx\n",
+					__func__, hba->outstanding_reqs);
+			break;
+		}
+	}
+
 	/* release lock as clear command might sleep */
drivers/scsi/ufs/ufshcd.c +7 -3
 	spin_unlock_irqrestore(hba->host->host_lock, flags);
 	/* Clear pending transfer requests */
@@ -6302,9 +6322,13 @@ static irqreturn_t ufshcd_intr(int irq, void
 *__hba)
 		intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 	}
 
-	if (enabled_intr_status && retval == IRQ_NONE) {
-		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x\n",
-					__func__, intr_status);
+	if (enabled_intr_status && retval == IRQ_NONE &&
+				!ufshcd_eh_in_progress(hba)) {
+		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x,
 0x%08x)\n",
+					__func__,
+					intr_status,
+					hba->ufs_stats.last_intr_status,
+					enabled_intr_status);
 		ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
drivers/scsi/ufs/ufshcd.c +4 -1
 	}
@@ -6348,7 +6372,10 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba
 *hba,
 	 * Even though we use wait_event() which sleeps indefinitely,
 	 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
 	 */
-	req = blk_get_request(q, REQ_OP_DRV_OUT, BLK_MQ_REQ_RESERVED);
+	req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
 	req->end_io_data = &wait;
 	free_slot = req->tag;

WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);

#8 lore raw

Re: [PATCH v4 2/2] scsi: ufs: handle LINERESET with correct tm_cmd

Quoted 43 lines · Jaegeuk Kim
Quoted 34 lines · Can Guo
Quoted 24 lines · Jaegeuk Kim
Quoted 23 lines · Can Guo
Quoted 13 lines · Jaegeuk Kim

From: Jaegeuk Kim <[email protected]>

This fixes a warning caused by wrong reserve tag usage in __ufshcd_issue_tm_cmd.

WARNING: CPU: 7 PID: 7 at block/blk-core.c:630 blk_get_request+0x68/0x70
WARNING: CPU: 4 PID: 157 at block/blk-mq-tag.c:82
blk_mq_get_tag+0x438/0x46c

And, in ufshcd_err_handler(), we can avoid to send tm_cmd before aborting
outstanding commands by waiting a bit for IO completion like this.

Quoted 6 lines

__ufshcd_issue_tm_cmd: task management cmd 0x80 timed-out

Fixes 69a6c269c097 ("scsi: ufs: Use blk_{get,put}_request() to

allocate and free TMFs")

Fixes 2355b66ed20c ("scsi: ufs: Handle LINERESET indication in err

handler")

Hi Jaegeuk,

Quoted 8 lines
Quoted 2 lines

Sorry, what is wrong with commit 2355b66ed20c? Clearing pending I/O reqs is a general procedure for handling all non-fatal errors.

Without waiting IOs, I hit the below timeout all the time from LINERESET, which
causes UFS stuck permanently, as mentioned in the description.

"__ufshcd_issue_tm_cmd: task management cmd 0x80 timed-out"

Quoted 7 lines

In that case, ufshcd_try_to_abort_task(), the caller of __ufshcd_issue_tm_cmd(),
should return -ETIMEOUT, then err_handler would jump to do a full reset, then bail.
I am not sure what gets UFS stuck permanently. Could you please share the callstack
if possible? I really want to know what is happening. Thanks.

Quoted 2 lines

I can't share all the log tho, it entered full reset. While printing out whole registers, the device was hard reset. Thanks,

Hi Jaegeuk,

Entering full reset is expected in this case, which is why I am saying line-reset handling logic should not be penalized. I think we need to find out what caused the hard reset but not just adding a delay before clearing pending reqs, because let's say 3 sec expires and you hit the same tm req timeout (maybe with a lower possibility), you may still end up same at the hard reset. You don't need to share all the log, just the last call stacks before hard reset. Is it a QCOM's platform used in your case? Can you check the log/dump if NoC error happened?

Thanks.
Can Guo.

Quoted 95 lines
Quoted 95 lines

Regards,
Can Guo.

Quoted 90 lines
Quoted 90 lines

Thanks,
Can Guo.

Quoted 85 lines
---
 drivers/scsi/ufs/ufshcd.c | 35 +++++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)
drivers/scsi/ufs/ufshcd.c +20 -0
index e6e7bdf99cd7..340dd5e515dd 100644
@@ -44,6 +44,9 @@
 /* Query request timeout */
 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
 
+/* LINERESET TIME OUT */
+#define LINERESET_IO_TIMEOUT_MS			(30000) /* 30 sec */
+
 /* Task management command timeout */
 #define TM_CMD_TIMEOUT	100 /* msecs */
 
@@ -5826,6 +5829,7 @@ static void ufshcd_err_handler(struct work_struct
 *work)
 	int err = 0, pmc_err;
 	int tag;
 	bool needs_reset = false, needs_restore = false;
+	ktime_t start;
 
 	hba = container_of(work, struct ufs_hba, eh_work);
@@ -5911,6 +5915,22 @@ static void ufshcd_err_handler(struct work_struct
 *work)
 	}
 
 	hba->silence_err_logs = true;
+
+	/* Wait for IO completion for non-fatal errors to avoid aborting IOs
 */
+	start = ktime_get();
+	while (hba->outstanding_reqs) {
+		ufshcd_complete_requests(hba);
+		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		schedule();
+		spin_lock_irqsave(hba->host->host_lock, flags);
+		if (ktime_to_ms(ktime_sub(ktime_get(), start)) >
+						LINERESET_IO_TIMEOUT_MS) {
+			dev_err(hba->dev, "%s: timeout, outstanding=0x%lx\n",
+					__func__, hba->outstanding_reqs);
+			break;
+		}
+	}
+
 	/* release lock as clear command might sleep */
drivers/scsi/ufs/ufshcd.c +7 -3
 	spin_unlock_irqrestore(hba->host->host_lock, flags);
 	/* Clear pending transfer requests */
@@ -6302,9 +6322,13 @@ static irqreturn_t ufshcd_intr(int irq, void
 *__hba)
 		intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 	}
 
-	if (enabled_intr_status && retval == IRQ_NONE) {
-		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x\n",
-					__func__, intr_status);
+	if (enabled_intr_status && retval == IRQ_NONE &&
+				!ufshcd_eh_in_progress(hba)) {
+		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x,
 0x%08x)\n",
+					__func__,
+					intr_status,
+					hba->ufs_stats.last_intr_status,
+					enabled_intr_status);
 		ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
drivers/scsi/ufs/ufshcd.c +4 -1
 	}
@@ -6348,7 +6372,10 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba
 *hba,
 	 * Even though we use wait_event() which sleeps indefinitely,
 	 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
 	 */
-	req = blk_get_request(q, REQ_OP_DRV_OUT, BLK_MQ_REQ_RESERVED);
+	req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
 	req->end_io_data = &wait;
 	free_slot = req->tag;

WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);

#9 lore raw

Re: [PATCH v4 2/2] scsi: ufs: handle LINERESET with correct tm_cmd

Quoted 58 lines · Can Guo
Quoted 43 lines · Jaegeuk Kim
Quoted 34 lines · Can Guo
Quoted 24 lines · Jaegeuk Kim
Quoted 23 lines · Can Guo
Quoted 13 lines · Jaegeuk Kim

From: Jaegeuk Kim <[email protected]>

This fixes a warning caused by wrong reserve tag usage in __ufshcd_issue_tm_cmd.

WARNING: CPU: 7 PID: 7 at block/blk-core.c:630 blk_get_request+0x68/0x70
WARNING: CPU: 4 PID: 157 at block/blk-mq-tag.c:82
blk_mq_get_tag+0x438/0x46c

And, in ufshcd_err_handler(), we can avoid to send tm_cmd before aborting
outstanding commands by waiting a bit for IO completion like this.

Quoted 6 lines

__ufshcd_issue_tm_cmd: task management cmd 0x80 timed-out

Fixes 69a6c269c097 ("scsi: ufs: Use blk_{get,put}_request() to

allocate and free TMFs")

Fixes 2355b66ed20c ("scsi: ufs: Handle LINERESET indication in err

handler")

Hi Jaegeuk,

Quoted 8 lines
Quoted 2 lines

Sorry, what is wrong with commit 2355b66ed20c? Clearing pending I/O reqs is a general procedure for handling all non-fatal errors.

Without waiting IOs, I hit the below timeout all the time from LINERESET, which
causes UFS stuck permanently, as mentioned in the description.

"__ufshcd_issue_tm_cmd: task management cmd 0x80 timed-out"

Quoted 7 lines

In that case, ufshcd_try_to_abort_task(), the caller of __ufshcd_issue_tm_cmd(),
should return -ETIMEOUT, then err_handler would jump to do a full reset, then bail.
I am not sure what gets UFS stuck permanently. Could you please share the callstack
if possible? I really want to know what is happening. Thanks.

Quoted 2 lines

I can't share all the log tho, it entered full reset. While printing out whole registers, the device was hard reset. Thanks,

Hi Jaegeuk,

Entering full reset is expected in this case, which is why I am saying line-reset handling logic should not be penalized. I think we need to find out what caused the hard reset but not just adding a delay before clearing pending reqs, because let's say 3 sec expires and you hit the same tm req timeout (maybe with a lower possibility), you may still end up same at the hard reset. You don't need to share all the log, just the last call stacks before hard reset. Is it a QCOM's platform used in your case? Can you check the log/dump if NoC error happened?

Quoted 3 lines

Thanks.
Can Guo.

Hi Jaegeuk,

If it is QCOM's platform, what you described looks like a known issue which we have already fixed in downstream. Please try attached patch. If not, please ignore it.

Thanks,
Can Guo.

Quoted 82 lines
Quoted 82 lines
Quoted 82 lines

Regards,
Can Guo.

Quoted 77 lines
Quoted 77 lines

Thanks,
Can Guo.

Quoted 72 lines
---
 drivers/scsi/ufs/ufshcd.c | 35 +++++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)
drivers/scsi/ufs/ufshcd.c +20 -0
index e6e7bdf99cd7..340dd5e515dd 100644
@@ -44,6 +44,9 @@
 /* Query request timeout */
 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
 
+/* LINERESET TIME OUT */
+#define LINERESET_IO_TIMEOUT_MS			(30000) /* 30 sec */
+
 /* Task management command timeout */
 #define TM_CMD_TIMEOUT	100 /* msecs */
 
@@ -5826,6 +5829,7 @@ static void ufshcd_err_handler(struct work_struct
 *work)
 	int err = 0, pmc_err;
 	int tag;
 	bool needs_reset = false, needs_restore = false;
+	ktime_t start;
 
 	hba = container_of(work, struct ufs_hba, eh_work);
@@ -5911,6 +5915,22 @@ static void ufshcd_err_handler(struct work_struct
 *work)
 	}
 
 	hba->silence_err_logs = true;
+
+	/* Wait for IO completion for non-fatal errors to avoid aborting IOs
 */
+	start = ktime_get();
+	while (hba->outstanding_reqs) {
+		ufshcd_complete_requests(hba);
+		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		schedule();
+		spin_lock_irqsave(hba->host->host_lock, flags);
+		if (ktime_to_ms(ktime_sub(ktime_get(), start)) >
+						LINERESET_IO_TIMEOUT_MS) {
+			dev_err(hba->dev, "%s: timeout, outstanding=0x%lx\n",
+					__func__, hba->outstanding_reqs);
+			break;
+		}
+	}
+
 	/* release lock as clear command might sleep */
drivers/scsi/ufs/ufshcd.c +7 -3
 	spin_unlock_irqrestore(hba->host->host_lock, flags);
 	/* Clear pending transfer requests */
@@ -6302,9 +6322,13 @@ static irqreturn_t ufshcd_intr(int irq, void
 *__hba)
 		intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 	}
 
-	if (enabled_intr_status && retval == IRQ_NONE) {
-		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x\n",
-					__func__, intr_status);
+	if (enabled_intr_status && retval == IRQ_NONE &&
+				!ufshcd_eh_in_progress(hba)) {
+		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x,
 0x%08x)\n",
+					__func__,
+					intr_status,
+					hba->ufs_stats.last_intr_status,
+					enabled_intr_status);
 		ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");

}

Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
drivers/scsi/ufs/ufshcd.c +0 -0
@@ -6348,7 +6372,10 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba
 *hba,
 	 * Even though we use wait_event() which sleeps indefinitely,
 	 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
 	 */
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
drivers/scsi/ufs/ufshcd.c +4 -1
-	req = blk_get_request(q, REQ_OP_DRV_OUT, BLK_MQ_REQ_RESERVED);
+	req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
 	req->end_io_data = &wait;
 	free_slot = req->tag;

WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);

From b50e595a80e9e31c2f9744439b3bdd986e9433ea Mon Sep 17 00:00:00 2001
From: Can Guo <[email protected]>
Date: Thu, 7 Jan 2021 01:07:12 -0800
Subject: [PATCH] scsi: ufs-qcom: Disable interrupt in reset path

Disable interrupt in reset path

drivers/scsi/ufs/ufs-qcom.c +10 -0
index 2206b1e..e55201f 100644
@@ -253,12 +253,17 @@ static int ufs_qcom_host_reset(struct ufs_hba *hba)
 {
 	int ret = 0;
 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);
+	bool reenable_intr = false;
 
 	if (!host->core_reset) {
 		dev_warn(hba->dev, "%s: reset control not set\n", __func__);
 		goto out;
 	}
 
+	reenable_intr = hba->is_irq_enabled;
+	disable_irq(hba->irq);
+	hba->is_irq_enabled = false;
+
 	ret = reset_control_assert(host->core_reset);
 	if (ret) {
 		dev_err(hba->dev, "%s: core_reset assert failed, err = %d\n",
@@ -280,6 +285,11 @@ static int ufs_qcom_host_reset(struct ufs_hba *hba)
 
 	usleep_range(1000, 1100);
 
+	if (reenable_intr) {
+		enable_irq(hba->irq);
+		hba->is_irq_enabled = true;
+	}
+
 out:
 	return ret;
 }
Signature · 1 line
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, a Linux Foundation Collaborative Project.
#10 lore raw

Re: [PATCH v4 2/2] scsi: ufs: handle LINERESET with correct tm_cmd

In reply to Can Guo
Quoted 51 lines · Can Guo
Quoted 45 lines · Jaegeuk Kim
Quoted 34 lines · Can Guo
Quoted 24 lines · Jaegeuk Kim
Quoted 23 lines · Can Guo
Quoted 13 lines · Jaegeuk Kim

From: Jaegeuk Kim <[email protected]>

This fixes a warning caused by wrong reserve tag usage in __ufshcd_issue_tm_cmd.

WARNING: CPU: 7 PID: 7 at block/blk-core.c:630 blk_get_request+0x68/0x70
WARNING: CPU: 4 PID: 157 at block/blk-mq-tag.c:82
blk_mq_get_tag+0x438/0x46c

And, in ufshcd_err_handler(), we can avoid to send tm_cmd before aborting
outstanding commands by waiting a bit for IO completion like this.

Quoted 6 lines

__ufshcd_issue_tm_cmd: task management cmd 0x80 timed-out

Fixes 69a6c269c097 ("scsi: ufs: Use blk_{get,put}_request() to

allocate and free TMFs")

Fixes 2355b66ed20c ("scsi: ufs: Handle LINERESET indication in err

handler")

Hi Jaegeuk,

Quoted 8 lines
Quoted 2 lines

Sorry, what is wrong with commit 2355b66ed20c? Clearing pending I/O reqs is a general procedure for handling all non-fatal errors.

Without waiting IOs, I hit the below timeout all the time from LINERESET, which
causes UFS stuck permanently, as mentioned in the description.

"__ufshcd_issue_tm_cmd: task management cmd 0x80 timed-out"

Quoted 9 lines

In that case, ufshcd_try_to_abort_task(), the caller of __ufshcd_issue_tm_cmd(),
should return -ETIMEOUT, then err_handler would jump to do a full reset,
then bail.
I am not sure what gets UFS stuck permanently. Could you please share the
callstack
if possible? I really want to know what is happening. Thanks.

Quoted 2 lines

I can't share all the log tho, it entered full reset. While printing out whole registers, the device was hard reset. Thanks,

Hi Jaegeuk,

Quoted 8 lines

Entering full reset is expected in this case, which is why I am saying line-reset handling logic should not be penalized. I think we need to find out what caused the hard reset but not just adding a delay before clearing pending reqs, because let's say 3 sec expires and you hit the same tm req timeout (maybe with a lower possibility), you may still end up same at the hard reset. You don't need to share all the log, just the last call stacks before hard reset. Is it a QCOM's platform used in your case? Can you check the log/dump if NoC error happened?

Hi Can,

I figured out it is caused by verbose kernel logs printed in terminal. I posted v5, so could you please review it?

Thanks,

Quoted 99 lines

Thanks.
Can Guo.

Quoted 95 lines
Quoted 95 lines

Regards,
Can Guo.

Quoted 90 lines
Quoted 90 lines

Thanks,
Can Guo.

Quoted 85 lines
---
 drivers/scsi/ufs/ufshcd.c | 35 +++++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)
drivers/scsi/ufs/ufshcd.c +20 -0
index e6e7bdf99cd7..340dd5e515dd 100644
@@ -44,6 +44,9 @@
 /* Query request timeout */
 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
 
+/* LINERESET TIME OUT */
+#define LINERESET_IO_TIMEOUT_MS			(30000) /* 30 sec */
+
 /* Task management command timeout */
 #define TM_CMD_TIMEOUT	100 /* msecs */
 
@@ -5826,6 +5829,7 @@ static void ufshcd_err_handler(struct work_struct
 *work)
 	int err = 0, pmc_err;
 	int tag;
 	bool needs_reset = false, needs_restore = false;
+	ktime_t start;
 
 	hba = container_of(work, struct ufs_hba, eh_work);
@@ -5911,6 +5915,22 @@ static void ufshcd_err_handler(struct work_struct
 *work)
 	}
 
 	hba->silence_err_logs = true;
+
+	/* Wait for IO completion for non-fatal errors to avoid aborting IOs
 */
+	start = ktime_get();
+	while (hba->outstanding_reqs) {
+		ufshcd_complete_requests(hba);
+		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		schedule();
+		spin_lock_irqsave(hba->host->host_lock, flags);
+		if (ktime_to_ms(ktime_sub(ktime_get(), start)) >
+						LINERESET_IO_TIMEOUT_MS) {
+			dev_err(hba->dev, "%s: timeout, outstanding=0x%lx\n",
+					__func__, hba->outstanding_reqs);
+			break;
+		}
+	}
+
 	/* release lock as clear command might sleep */
drivers/scsi/ufs/ufshcd.c +7 -3
 	spin_unlock_irqrestore(hba->host->host_lock, flags);
 	/* Clear pending transfer requests */
@@ -6302,9 +6322,13 @@ static irqreturn_t ufshcd_intr(int irq, void
 *__hba)
 		intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 	}
 
-	if (enabled_intr_status && retval == IRQ_NONE) {
-		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x\n",
-					__func__, intr_status);
+	if (enabled_intr_status && retval == IRQ_NONE &&
+				!ufshcd_eh_in_progress(hba)) {
+		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x,
 0x%08x)\n",
+					__func__,
+					intr_status,
+					hba->ufs_stats.last_intr_status,
+					enabled_intr_status);
 		ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
drivers/scsi/ufs/ufshcd.c +4 -1
 	}
@@ -6348,7 +6372,10 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba
 *hba,
 	 * Even though we use wait_event() which sleeps indefinitely,
 	 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
 	 */
-	req = blk_get_request(q, REQ_OP_DRV_OUT, BLK_MQ_REQ_RESERVED);
+	req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
 	req->end_io_data = &wait;
 	free_slot = req->tag;

WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);