Fix SCSI async abort handling when eh_deadline is active
v2 cover Fix SCSI async abort handling when eh_deadline is active
There is a code path in the SCSI async abort handling that can cause error handling of subsequent scsi_cmnds to proceed immediately to host reset with no other attempt at recovery.
This can be seen by the following:
modprobe scsi_debug every_nth=10 opts=4
echo 7 > /sys/module/scsi_mod/parameters/scsi_logging_level echo 10 > /sys/devices/pseudo_0/adapter0/host8/scsi_host/host<N>/eh_deadline
and performing I/O to the scsi_debug device, the host will get reset because prior aborts succeeded, because ->last_reset does not get invalidated.
The patch series contains a fix, followed by a simplification of the control flow to remove duplicate code. Only the first patch is Cc: stable as the second part doesn't qualify. Yes, I know the first patch is >100 lines, I couldn't make it smaller unfortunately.
v2:
- Introduced scsi_eh_abort_cleanup() in patch 1/1 to factor out code (This is then removed in patch 2/2 since code refactoring results in only one place it is called though.)
- Moved introduction of local "shost" to cleanup patch 2/2
Ewan D. Milne (2):
scsi: core: avoid leaving shost->last_reset with stale value if EH
does not run
scsi: core: simplify control flow in scmd_eh_abort_handler()
drivers/scsi/hosts.c | 1 + drivers/scsi/scsi_error.c | 92 ++++++++++++++++++++++++++++++----------------- drivers/scsi/scsi_lib.c | 1 + include/scsi/scsi_cmnd.h | 2 +- include/scsi/scsi_host.h | 1 + 5 files changed, 63 insertions(+), 34 deletions(-)
Signature · 1 line
2.1.0
PATCH v2 1/2 scsi: core: avoid leaving shost->last_reset with stale value if EH does not run
The changes to issue the abort from the scmd->abort_work instead of the EH thread introduced a problem if eh_deadline is used. If aborting the command(s) is successful, and there are never any scmds added to the shost->eh_cmd_q, there is no code path which will reset the ->last_reset value back to zero.
The effect of this is that after a successful abort with no EH thread activity, a subsequent timeout, perhaps a long time later, might immediately be considered past a user-set eh_deadline time, and the host will be reset with no attempt at recovery.
Fix this by resetting ->last_reset back to zero in scmd_eh_abort_handler() if it is determined that the EH thread will not run to do this.
Thanks to Gopinath Marappan for investigating this problem.
--- drivers/scsi/hosts.c | 1 + drivers/scsi/scsi_error.c | 25 +++++++++++++++++++++++++ drivers/scsi/scsi_lib.c | 1 + include/scsi/scsi_cmnd.h | 2 +- include/scsi/scsi_host.h | 1 + 5 files changed, 29 insertions(+), 1 deletion(-)
drivers/scsi/hosts.c +1 -0
@@ -387,6 +387,7 @@ struct Scsi_Host *scsi_host_alloc(struct scsi_host_template *sht, int privsize) shost->shost_state =SHOST_CREATED ;INIT_LIST_HEAD (&shost->__devices);INIT_LIST_HEAD (&shost->__targets); +INIT_LIST_HEAD (&shost->eh_abort_list);INIT_LIST_HEAD (&shost->eh_cmd_q);INIT_LIST_HEAD (&shost->starved_list);init_waitqueue_head (&shost->host_wait);
drivers/scsi/scsi_error.c +25 -0
@@ -135,6 +135,23 @@ static bool scsi_eh_should_retry_cmd(struct scsi_cmnd *cmd)return true; } +static void scsi_eh_complete_abort (struct scsi_cmnd *scmd,struct Scsi_Host *shost) +{ +unsigned long flags; + +spin_lock_irqsave (shost->host_lock, flags); +list_del_init (&scmd->eh_entry); +/* +* If the abort succeeds, and there is no further +* EH action, clear the ->last_reset time. +*/ +if (list_empty (&shost->eh_abort_list) && +list_empty (&shost->eh_cmd_q)) +if (shost->eh_deadline !=-1 ) + shost->last_reset =0 ; +spin_unlock_irqrestore (shost->host_lock, flags); +} +/** * scmd_eh_abort_handler - Handle command aborts * @work: command to be aborted.
@@ -152,6 +169,7 @@ scmd_eh_abort_handler(struct work_struct *work) container_of(work, struct scsi_cmnd, abort_work.work);struct scsi_device *sdev = scmd->device;enum scsi_disposition rtn; +unsigned long flags;if (scsi_host_eh_past_deadline (sdev->host)) { SCSI_LOG_ERROR_RECOVERY(3,
@@ -175,12 +193,14 @@ scmd_eh_abort_handler(struct work_struct *work)SCSI_LOG_ERROR_RECOVERY (3 ,scmd_printk (KERN_WARNING , scmd,"retry aborted command\n" )); +scsi_eh_complete_abort (scmd, sdev->host);scsi_queue_insert (scmd,SCSI_MLQUEUE_EH_RETRY );return ; }else {SCSI_LOG_ERROR_RECOVERY (3 ,scmd_printk (KERN_WARNING , scmd,"finish aborted command\n" )); +scsi_eh_complete_abort (scmd, sdev->host);scsi_finish_command (scmd);return ; }
@@ -193,6 +213,9 @@ scmd_eh_abort_handler(struct work_struct *work) } } +spin_lock_irqsave (sdev->host->host_lock, flags); +list_del_init (&scmd->eh_entry); +spin_unlock_irqrestore (sdev->host->host_lock, flags);scsi_eh_scmd_add (scmd); }
@@ -223,6 +246,8 @@ scsi_abort_command(struct scsi_cmnd *scmd)spin_lock_irqsave (shost->host_lock, flags);if (shost->eh_deadline !=-1 && !shost->last_reset) shost->last_reset = jiffies; +BUG_ON (!list_empty (&scmd->eh_entry)); +list_add_tail (&scmd->eh_entry, &shost->eh_abort_list);spin_unlock_irqrestore (shost->host_lock, flags); scmd->eh_eflags |=SCSI_EH_ABORT_SCHEDULED ;
drivers/scsi/scsi_lib.c +1 -0
@@ -1143,6 +1143,7 @@ void scsi_init_command(struct scsi_device *dev, struct scsi_cmnd *cmd) cmd->sense_buffer = buf; cmd->prot_sdb = prot; cmd->flags = flags; +INIT_LIST_HEAD (&cmd->eh_entry);INIT_DELAYED_WORK (&cmd->abort_work, scmd_eh_abort_handler); cmd->jiffies_at_alloc = jiffies_at_alloc; cmd->retries = retries;
include/scsi/scsi_cmnd.h +1 -1
@@ -68,7 +68,7 @@ struct scsi_pointer {struct scsi_cmnd {struct scsi_request req;struct scsi_device *device; -struct list_head eh_entry;/* entry for the host eh_cmd_q */ +struct list_head eh_entry;/* entry for the host eh_abort_list/eh_cmd_q */ struct delayed_work abort_work;struct rcu_head rcu;
include/scsi/scsi_host.h +1 -0
@@ -556,6 +556,7 @@ struct Scsi_Host {struct mutex scan_mutex;/* serialize scanning activity */ +struct list_head eh_abort_list;struct list_head eh_cmd_q;struct task_struct * ehandler;/* Error recovery thread. */ struct completion * eh_action;/* Wait for specific actions on the
Signature · 1 line
2.1.0
PATCH v2 2/2 scsi: core: simplify control flow in scmd_eh_abort_handler()
In reply to Ewan D. MilneSimplify the nested conditionals in the function by using a label for the error path. Introduce local "shost" to avoid repeated "sdev->shost" usage. Also remove scsi_eh_complete_abort() since there is now only one place it would be called.
--- drivers/scsi/scsi_error.c | 109 +++++++++++++++++++++++----------------------- 1 file changed, 55 insertions(+), 54 deletions(-)
drivers/scsi/scsi_error.c +55 -54
@@ -135,23 +135,6 @@ static bool scsi_eh_should_retry_cmd(struct scsi_cmnd *cmd)return true; } -static void scsi_eh_complete_abort (struct scsi_cmnd *scmd,struct Scsi_Host *shost) -{ -unsigned long flags; - -spin_lock_irqsave (shost->host_lock, flags); -list_del_init (&scmd->eh_entry); -/* -* If the abort succeeds, and there is no further -* EH action, clear the ->last_reset time. -*/ -if (list_empty (&shost->eh_abort_list) && -list_empty (&shost->eh_cmd_q)) -if (shost->eh_deadline !=-1 ) - shost->last_reset =0 ; -spin_unlock_irqrestore (shost->host_lock, flags); -} -/** * scmd_eh_abort_handler - Handle command aborts * @work: command to be aborted.
@@ -168,54 +151,72 @@ scmd_eh_abort_handler(struct work_struct *work)struct scsi_cmnd *scmd = container_of(work, struct scsi_cmnd, abort_work.work);struct scsi_device *sdev = scmd->device; +struct Scsi_Host *shost = sdev->host;enum scsi_disposition rtn;unsigned long flags; -if (scsi_host_eh_past_deadline (sdev->host)) { +if (scsi_host_eh_past_deadline (shost)) {SCSI_LOG_ERROR_RECOVERY (3 ,scmd_printk (KERN_INFO , scmd,"eh timeout, not aborting\n" )); - }else { -SCSI_LOG_ERROR_RECOVERY (3 , + goto out; + } + +SCSI_LOG_ERROR_RECOVERY (3 ,scmd_printk (KERN_INFO , scmd,"aborting command\n" )); - rtn =scsi_try_to_abort_cmd (sdev->host->hostt, scmd); -if (rtn ==SUCCESS ) { -set_host_byte (scmd,DID_TIME_OUT ); -if (scsi_host_eh_past_deadline (sdev->host)) { -SCSI_LOG_ERROR_RECOVERY (3 , -scmd_printk (KERN_INFO , scmd, -"eh timeout, not retrying " -"aborted command\n" )); - }else if (!scsi_noretry_cmd (scmd) && -scsi_cmd_retry_allowed (scmd) && -scsi_eh_should_retry_cmd (scmd)) { -SCSI_LOG_ERROR_RECOVERY (3 , -scmd_printk (KERN_WARNING , scmd, -"retry aborted command\n" )); -scsi_eh_complete_abort (scmd, sdev->host); -scsi_queue_insert (scmd,SCSI_MLQUEUE_EH_RETRY ); -return ; - }else { -SCSI_LOG_ERROR_RECOVERY (3 , -scmd_printk (KERN_WARNING , scmd, -"finish aborted command\n" )); -scsi_eh_complete_abort (scmd, sdev->host); -scsi_finish_command (scmd); -return ; - } - }else { -SCSI_LOG_ERROR_RECOVERY (3 , -scmd_printk (KERN_INFO , scmd, -"cmd abort %s\n" , - (rtn ==FAST_IO_FAIL ) ? -"not send" :"failed" )); - } + rtn =scsi_try_to_abort_cmd (shost->hostt, scmd); +if (rtn !=SUCCESS ) { +SCSI_LOG_ERROR_RECOVERY (3 , +scmd_printk (KERN_INFO , scmd, +"cmd abort %s\n" , + (rtn ==FAST_IO_FAIL ) ? +"not send" :"failed" )); + goto out; + } +set_host_byte (scmd,DID_TIME_OUT ); +if (scsi_host_eh_past_deadline (shost)) { +SCSI_LOG_ERROR_RECOVERY (3 , +scmd_printk (KERN_INFO , scmd, +"eh timeout, not retrying " +"aborted command\n" )); + goto out; } -spin_lock_irqsave (sdev->host->host_lock, flags); +spin_lock_irqsave (shost->host_lock, flags);list_del_init (&scmd->eh_entry); -spin_unlock_irqrestore (sdev->host->host_lock, flags); + +/* +* If the abort succeeds, and there is no further +* EH action, clear the ->last_reset time. +*/ +if (list_empty (&shost->eh_abort_list) && +list_empty (&shost->eh_cmd_q)) +if (shost->eh_deadline !=-1 ) + shost->last_reset =0 ; + +spin_unlock_irqrestore (shost->host_lock, flags); + +if (!scsi_noretry_cmd (scmd) && +scsi_cmd_retry_allowed (scmd) && +scsi_eh_should_retry_cmd (scmd)) { +SCSI_LOG_ERROR_RECOVERY (3 , +scmd_printk (KERN_WARNING , scmd, +"retry aborted command\n" )); +scsi_queue_insert (scmd,SCSI_MLQUEUE_EH_RETRY ); + }else { +SCSI_LOG_ERROR_RECOVERY (3 , +scmd_printk (KERN_WARNING , scmd, +"finish aborted command\n" )); +scsi_finish_command (scmd); + } +return ; + +out: +spin_lock_irqsave (shost->host_lock, flags); +list_del_init (&scmd->eh_entry); +spin_unlock_irqrestore (shost->host_lock, flags); +scsi_eh_scmd_add (scmd); }
Signature · 1 line
2.1.0