New EH command timeout handler
Subsystems (5)
v4 cover New EH command timeout handler
his patchset implements a new SCSI EH command timeout handler which will be sending command aborts inline without actually engaging SCSI EH.
SCSI EH will only be invoked if command abort fails.
In addition the commands will be returned directly if the command abort succeeded, cutting down recovery times dramatically.
With the original SCSI EH I got:
# time dd if=/dev/zero of=/dev/dm-2 bs=4k count=4k oflag=direct 4096+0 records in 4096+0 records out 16777216 bytes (17 MB) copied, 142.652 s, 118 kB/s
real 2m22.657s
user 0m0.013s
sys 0m0.145s
With this patchset I got:
# time dd if=/dev/zero of=/dev/dm-2 bs=4k count=4k oflag=direct 4096+0 records in 4096+0 records out 16777216 bytes (17 MB) copied, 52.1579 s, 322 kB/s
real 0m52.163s
user 0m0.012s
sys 0m0.145s
Test was to disable RSCN on the target port, disable the target port, and then start the 'dd' command as indicated.
Changes to the original version:
- Use a private list in scsi_eh_abort_handler to avoid list starvation (pointed out by Joern Engel)
- Terminate command aborts when the first abort fails
- Do not attempt command aborts if the host is already in recovery or if the device is removed.
- Flush abort workqueue if the device is removed.
Changes to v2:
- Removed eh_entry initialisation
- Convert to per-command workqueue
Changes to v3:
- Use delayed_work
- Enable new eh timeout handler for virtio, SAS, and FC
- Modify logging messages to include scmd pointer
Hannes Reinecke (9):
scsi: Fix erratic device offline during EH
blk-timeout: add BLK_EH_SCHEDULED return code
scsi: improved eh timeout handler
virtio_scsi: Enable new EH timeout handler
libsas: Enable new EH timeout handler
mptsas: Enable new EH timeout handler
mpt2sas: Enable new EH timeout handler
mpt3sas: Enable new EH timeout handler
scsi_transport_fc: Enable new EH timeout handler
drivers/message/fusion/mptsas.c | 3 +- drivers/message/fusion/mptscsih.c | 7 ++ drivers/message/fusion/mptscsih.h | 1 + drivers/scsi/libsas/sas_scsi_host.c | 2 +- drivers/scsi/mpt2sas/mpt2sas_scsih.c | 13 ++- drivers/scsi/mpt3sas/mpt3sas_scsih.c | 11 +++ drivers/scsi/scsi.c | 3 + drivers/scsi/scsi_error.c | 178 ++++++++++++++++++++++++++++++----- drivers/scsi/scsi_priv.h | 2 + drivers/scsi/scsi_transport_fc.c | 2 +- drivers/scsi/virtio_scsi.c | 8 ++ include/linux/blkdev.h | 1 + include/scsi/scsi_cmnd.h | 2 + 13 files changed, 207 insertions(+), 26 deletions(-)
Signature · 1 line
1.7.12.4
PATCH v4 1/9 scsi: Fix erratic device offline during EH
Commit 18a4d0a22ed6c54b67af7718c305cd010f09ddf8
(Handle disk devices which can not process medium access commands) was introduced to offline any device which cannot process medium access commands.
However, commit 3eef6257de48ff84a5d98ca533685df8a3beaeb8 (Reduce error recovery time by reducing use of TURs) reduced the number of TURs by sending it only on the first failing command, which might or might not be a medium access command. So in combination this results in an erratic device offlining during EH; if the command where the TUR was sent upon happens to be a medium access command the device will be set offline, if not everything proceeds as normal.
So instead of checking the EH command in the ->eh_action callback we should rather call ->eh_action when we're about to finish the command _and_ have sent a TUR previously. This should then set the device offline as advertised.
--- drivers/scsi/scsi_error.c | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-)
drivers/scsi/scsi_error.c +19 -9
@@ -941,12 +941,6 @@ retry:scsi_eh_restore_cmnd (scmd, &ses); -if (scmd->request->cmd_type !=REQ_TYPE_BLOCK_PC ) { -struct scsi_driver *sdrv =scsi_cmd_to_driver (scmd); -if (sdrv->eh_action) - rtn = sdrv->eh_action (scmd, cmnd, cmnd_size, rtn); - } -return rtn; }
@@ -964,6 +958,18 @@ static int scsi_request_sense(struct scsi_cmnd *scmd)return scsi_send_eh_cmnd (scmd,NULL ,0 , scmd->device->eh_timeout, ~0 ); } +static int scsi_eh_action (struct scsi_cmnd *scmd,int rtn) +{ +static unsigned char tur_command[6 ] = {TEST_UNIT_READY ,0 ,0 ,0 ,0 ,0 }; + +if (scmd->request->cmd_type !=REQ_TYPE_BLOCK_PC ) { +struct scsi_driver *sdrv =scsi_cmd_to_driver (scmd); +if (sdrv->eh_action) + rtn = sdrv->eh_action (scmd, tur_command,6 , rtn); + } +return rtn; +} +/** * scsi_eh_finish_cmd - Handle a cmd that eh is finished with. * @scmd: Original SCSI cmd that eh has finished.
@@ -1142,7 +1148,9 @@ static int scsi_eh_test_devices(struct list_head *cmd_list,list_for_each_entry_safe (scmd, next, cmd_list, eh_entry)if (scmd->device == sdev) { -if (finish_cmds) +if (finish_cmds && + (try_stu || +scsi_eh_action (scmd,SUCCESS ) ==SUCCESS ))scsi_eh_finish_cmd (scmd, done_q);else list_move_tail (&scmd->eh_entry, work_q);
@@ -1281,7 +1289,8 @@ static int scsi_eh_stu(struct Scsi_Host *shost, !scsi_eh_tur(stu_scmd)) {list_for_each_entry_safe (scmd, next, work_q, eh_entry) { -if (scmd->device == sdev) +if (scmd->device == sdev && +scsi_eh_action (scmd,SUCCESS ) ==SUCCESS )scsi_eh_finish_cmd (scmd, done_q); } }
@@ -1348,7 +1357,8 @@ static int scsi_eh_bus_device_reset(struct Scsi_Host *shost, !scsi_eh_tur(bdr_scmd)) {list_for_each_entry_safe (scmd, next, work_q, eh_entry) { -if (scmd->device == sdev) +if (scmd->device == sdev && +scsi_eh_action (scmd, rtn) !=FAILED )scsi_eh_finish_cmd (scmd, done_q); }
Signature · 1 line
1.7.12.4
PATCH v4 2/9 blk-timeout: add BLK_EH_SCHEDULED return code
In reply to Hannes ReineckeAdd a 'BLK_EH_SCHEDULED' return code for blk-timeout to indicate that a delayed error recovery has been initiated.
--- drivers/scsi/scsi_error.c | 4 ++++ include/linux/blkdev.h | 1 + 2 files changed, 5 insertions(+)
drivers/scsi/scsi_error.c +4 -0
@@ -161,6 +161,10 @@ enum blk_eh_timer_return scsi_times_out(struct request *req)else if (host->hostt->eh_timed_out) rtn = host->hostt->eh_timed_out (scmd); +/* Check for asynchronous command aborts */ +if (rtn ==BLK_EH_SCHEDULED ) +return BLK_EH_NOT_HANDLED ; + scmd->result |=DID_TIME_OUT <<16 ; if (unlikely(rtn == BLK_EH_NOT_HANDLED &&
include/linux/blkdev.h +1 -0
@@ -238,6 +238,7 @@ enum blk_eh_timer_return {BLK_EH_NOT_HANDLED ,BLK_EH_HANDLED ,BLK_EH_RESET_TIMER , + BLK_EH_SCHEDULED, };typedef enum blk_eh_timer_return (rq_timed_out_fn )(struct request *);
Signature · 1 line
1.7.12.4
PATCH v4 3/9 scsi: improved eh timeout handler
In reply to Hannes ReineckeWhen a command runs into a timeout we need to send an 'ABORT TASK' TMF. This is typically done by the 'eh_abort_handler' LLDD callback.
Conceptually, however, this function is a normal SCSI command, so there is no need to enter the error handler.
This patch implements a new scsi_abort_command() function which invokes an asynchronous function scsi_eh_abort_handler() to abort the commands via the usual 'eh_abort_handler'.
If abort succeeds the command is either retried or terminated, depending on the number of allowed retries. However, 'eh_eflags' records the abort, so if the retry would fail again the command is pushed onto the error handler without trying to abort it (again); it'll be cleared up from SCSI EH.
--- drivers/scsi/scsi.c | 3 + drivers/scsi/scsi_error.c | 146 +++++++++++++++++++++++++++++++++++++++++----- drivers/scsi/scsi_priv.h | 2 + include/scsi/scsi_cmnd.h | 2 + 4 files changed, 140 insertions(+), 13 deletions(-)
drivers/scsi/scsi.c +3 -0
@@ -297,6 +297,7 @@ struct scsi_cmnd *scsi_get_command(struct scsi_device *dev, gfp_t gfp_mask) cmd->device = dev;INIT_LIST_HEAD (&cmd->list); +INIT_DELAYED_WORK (&cmd->abort_work, scmd_eh_abort_handler);spin_lock_irqsave (&dev->list_lock, flags);list_add_tail (&cmd->list, &dev->cmd_list);spin_unlock_irqrestore (&dev->list_lock, flags);
@@ -353,6 +354,8 @@ void scsi_put_command(struct scsi_cmnd *cmd)list_del_init (&cmd->list);spin_unlock_irqrestore (&cmd->device->list_lock, flags); +cancel_delayed_work (&cmd->abort_work); +__scsi_put_command (cmd->device->host, cmd, &sdev->sdev_gendev); }EXPORT_SYMBOL (scsi_put_command);
drivers/scsi/scsi_error.c +133 -13
191 lines not on this page · raw message
drivers/scsi/scsi_priv.h +2 -0
@@ -19,6 +19,7 @@ struct scsi_nl_hdr;* Scsi Error Handler Flags */ #define SCSI_EH_CANCEL_CMD 0x0001/* Cancel this cmd */ +#define SCSI_EH_ABORT_SCHEDULED 0x0002/* Abort has been scheduled */ #define SCSI_SENSE_VALID (scmd) \ (((scmd)->sense_buffer[0] & 0x70) == 0x70)
@@ -66,6 +67,7 @@ extern int __init scsi_init_devinfo(void);extern void scsi_exit_devinfo (void );/* scsi_error.c */ +extern void scmd_eh_abort_handler (struct work_struct *work);extern enum blk_eh_timer_return scsi_times_out (struct request *req);extern int scsi_error_handler (void *host);extern int scsi_decide_disposition (struct scsi_cmnd *cmd);
include/scsi/scsi_cmnd.h +2 -0
@@ -55,6 +55,7 @@ struct scsi_cmnd {struct scsi_device *device;struct list_head list;/* scsi_cmnd participates in queue lists */ struct list_head eh_entry;/* entry for the host eh_cmd_q */ +struct delayed_work abort_work;int eh_eflags;/* Used by error handlr */ /*
@@ -144,6 +145,7 @@ extern void scsi_put_command(struct scsi_cmnd *);extern void __scsi_put_command (struct Scsi_Host *,struct scsi_cmnd *,struct device *);extern void scsi_finish_command (struct scsi_cmnd *cmd); +extern enum blk_eh_timer_return scsi_abort_command (struct scsi_cmnd *cmd);extern void *scsi_kmap_atomic_sg (struct scatterlist *sg,int sg_count,size_t *offset,size_t *len);
Signature · 1 line
1.7.12.4
PATCH v4 4/9 virtio_scsi: Enable new EH timeout handler
In reply to Hannes Reinecke--- drivers/scsi/virtio_scsi.c | 8 ++++++++ 1 file changed, 8 insertions(+)
drivers/scsi/virtio_scsi.c +8 -0
@@ -630,6 +630,12 @@ static int virtscsi_device_reset(struct scsi_cmnd *sc)return virtscsi_tmf (vscsi, cmd); } +static enum blk_eh_timer_return virtscsi_timedout (struct scsi_cmnd *scmd) +{ +scsi_abort_command (scmd); +return BLK_EH_SCHEDULED ; +} +static int virtscsi_abort (struct scsi_cmnd *sc) {struct virtio_scsi *vscsi =shost_priv (sc->device->host);
@@ -683,6 +689,7 @@ static struct scsi_host_template virtscsi_host_template_single = { .queuecommand = virtscsi_queuecommand_single, .eh_abort_handler = virtscsi_abort, .eh_device_reset_handler = virtscsi_device_reset, + .eh_timed_out = virtscsi_timedout, .can_queue = 1024, .dma_boundary = UINT_MAX,
@@ -699,6 +706,7 @@ static struct scsi_host_template virtscsi_host_template_multi = { .queuecommand = virtscsi_queuecommand_multi, .eh_abort_handler = virtscsi_abort, .eh_device_reset_handler = virtscsi_device_reset, + .eh_timed_out = virtscsi_timedout, .can_queue = 1024, .dma_boundary = UINT_MAX,
Signature · 1 line
1.7.12.4
Re: [PATCH 4/9] virtio_scsi: Enable new EH timeout handler
drivers/scsi/virtio_scsi.c static int virtscsi_device_reset(struct scsi_cmnd *sc) · line 637
Quoted 6 lines
drivers/scsi/virtio_scsi.c +5
-0+static enum blk_eh_timer_return virtscsi_timedout (struct scsi_cmnd *scmd) +{ +scsi_abort_command (scmd); +return BLK_EH_SCHEDULED ; +}
just set the method vector to scsi_abort_command here and in all the other trivial conversions instead of having all those pointless wrappers.
Re: [PATCH 4/9] virtio_scsi: Enable new EH timeout handler
Quoted 11 lines · Christoph Hellwig
drivers/scsi/virtio_scsi.c static int virtscsi_device_reset(struct scsi_cmnd *sc) · line 637
Quoted 6 lines
drivers/scsi/virtio_scsi.c +5
-0+static enum blk_eh_timer_return virtscsi_timedout (struct scsi_cmnd *scmd) +{ +scsi_abort_command (scmd); +return BLK_EH_SCHEDULED ; +}just set the method vector to
scsi_abort_commandhere and in all the other trivial conversions instead of having all those pointless wrappers.
Ok.
Cheers,
Hannes
Signature · 8 lines
Dr. Hannes Reinecke zSeries & Storage [email protected] +49 911 74053 688 SUSE LINUX Products GmbH, Maxfeldstr. 5, 90409 Nürnberg GF: J. Hawn, J. Guild, F. Imendörffer, HRB 16746 (AG Nürnberg) -- To unsubscribe from this list: send the line "unsubscribe linux-scsi" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH v4 5/9 libsas: Enable new EH timeout handler
In reply to Hannes Reinecke--- drivers/scsi/libsas/sas_scsi_host.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/scsi/libsas/sas_scsi_host.c +1 -1
@@ -864,7 +864,7 @@ enum blk_eh_timer_return sas_scsi_timed_out(struct scsi_cmnd *cmd) {scmd_printk (KERN_DEBUG , cmd,"command %p timed out\n" , cmd); -return BLK_EH_NOT_HANDLED ; +return scsi_abort_command (cmd); }int sas_ioctl (struct scsi_device *sdev,int cmd,void __user *arg)
Signature · 1 line
1.7.12.4
PATCH v4 6/9 mptsas: Enable new EH timeout handler
In reply to Hannes Reinecke--- drivers/message/fusion/mptsas.c | 3 ++- drivers/message/fusion/mptscsih.c | 7 +++++++ drivers/message/fusion/mptscsih.h | 1 + 3 files changed, 10 insertions(+), 1 deletion(-)
drivers/message/fusion/mptsas.c +2 -1
@@ -1986,7 +1986,8 @@ static struct scsi_host_template mptsas_driver_template = { .slave_configure = mptsas_slave_configure, .target_destroy = mptsas_target_destroy, .slave_destroy = mptscsih_slave_destroy, - .change_queue_depth = mptscsih_change_queue_depth, + .change_queue_depth = mptscsih_change_queue_depth, + .eh_timed_out = mptscsih_timed_out, .eh_abort_handler = mptscsih_abort, .eh_device_reset_handler = mptscsih_dev_reset, .eh_host_reset_handler = mptscsih_host_reset,
drivers/message/fusion/mptscsih.c +7 -0
@@ -1679,6 +1679,13 @@ mptscsih_get_tm_timeout(MPT_ADAPTER *ioc) } } +enum blk_eh_timer_return +mptscsih_timed_out (struct scsi_cmnd *SCpnt) +{ +return scsi_abort_command (SCpnt); +} +EXPORT_SYMBOL (mptscsih_timed_out); +/*=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=*/ /** * mptscsih_abort - Abort linux scsi_cmnd routine, new_eh variant
drivers/message/fusion/mptscsih.h +1 -0
@@ -118,6 +118,7 @@ extern int mptscsih_IssueTaskMgmt(MPT_SCSI_HOST *hd, u8 type, u8 channel, u8 id, int lun, int ctx2abort, ulong timeout);extern void mptscsih_slave_destroy (struct scsi_device *device);extern int mptscsih_slave_configure (struct scsi_device *device); +enum blk_eh_timer_return mptscsih_timed_out (struct scsi_cmnd *SCpnt);extern int mptscsih_abort (struct scsi_cmnd * SCpnt);extern int mptscsih_dev_reset (struct scsi_cmnd * SCpnt);extern int mptscsih_bus_reset (struct scsi_cmnd * SCpnt);
Signature · 1 line
1.7.12.4
PATCH v4 7/9 mpt2sas: Enable new EH timeout handler
In reply to Hannes Reinecke--- drivers/scsi/mpt2sas/mpt2sas_scsih.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-)
drivers/scsi/mpt2sas/mpt2sas_scsih.c +12 -1
@@ -2569,6 +2569,16 @@ _scsih_tm_display_info(struct MPT2SAS_ADAPTER *ioc, struct scsi_cmnd *scmd) }/** +* _scsih_timed_out - eh timeout handler +* @scmd: pointer to scsi command object +*/ +static enum blk_eh_timer_return +_scsih_timed_out (struct scsi_cmnd *scmd) +{ +return scsi_abort_command (scmd); +} + +/** * _scsih_abort - eh threads main abort routine * @scmd: pointer to scsi command object *
@@ -7628,8 +7638,9 @@ static struct scsi_host_template scsih_driver_template = { .slave_destroy = _scsih_slave_destroy, .scan_finished = _scsih_scan_finished, .scan_start = _scsih_scan_start, - .change_queue_depth = _scsih_change_queue_depth, + .change_queue_depth = _scsih_change_queue_depth, .change_queue_type = _scsih_change_queue_type, + .eh_timed_out = _scsih_timed_out, .eh_abort_handler = _scsih_abort, .eh_device_reset_handler = _scsih_dev_reset, .eh_target_reset_handler = _scsih_target_reset,
Signature · 1 line
1.7.12.4
PATCH v4 8/9 mpt3sas: Enable new EH timeout handler
In reply to Hannes Reinecke--- drivers/scsi/mpt3sas/mpt3sas_scsih.c | 11 +++++++++++ 1 file changed, 11 insertions(+)
drivers/scsi/mpt3sas/mpt3sas_scsih.c +11 -0
@@ -2243,6 +2243,16 @@ _scsih_tm_display_info(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd) }/** +* _scsih_timed_out - eh timeout handler +* @scmd: pointer to scsi command object +*/ +static enum blk_eh_timer_return +_scsih_timed_out (struct scsi_cmnd *scmd) +{ +return scsi_abort_command (scmd); +} + +/** * _scsih_abort - eh threads main abort routine * @scmd: pointer to scsi command object *
@@ -7237,6 +7247,7 @@ static struct scsi_host_template scsih_driver_template = { .scan_start = _scsih_scan_start, .change_queue_depth = _scsih_change_queue_depth, .change_queue_type = _scsih_change_queue_type, + .eh_timed_out = _scsih_timed_out, .eh_abort_handler = _scsih_abort, .eh_device_reset_handler = _scsih_dev_reset, .eh_target_reset_handler = _scsih_target_reset,
Signature · 1 line
1.7.12.4
PATCH v4 9/9 scsi_transport_fc: Enable new EH timeout handler
In reply to Hannes Reinecke--- drivers/scsi/scsi_transport_fc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/scsi/scsi_transport_fc.c +1 -1
@@ -2079,7 +2079,7 @@ fc_timed_out(struct scsi_cmnd *scmd)if (rport->port_state ==FC_PORTSTATE_BLOCKED )return BLK_EH_RESET_TIMER ; -return BLK_EH_NOT_HANDLED ; +return scsi_abort_command (scmd); }/*
Signature · 1 line
1.7.12.4