RFC: add transport host byte values and common failure behavior
Subsystems (3)
iSCSI and FC classes and drivers have several similarities in how they handle transport problems. But between them and among the drivers there are also several differences, all of which make configuring them a little more fun they they need to be. The following patches attempt to begin to implement some common behavior. The iscsi parts are done in these patches since there are only three drivers and we all did sometihng similar already. FC is more fun and I did not modify all the drivers but I did modify the transport class
([PATCH 2/4] scsi: add transport host byte errors has more info).
The first patch can be ignored by most people. It is just some iscsi reworking to bring hw and sw iscsi closer together.
The second patch adds the new transport errors
The last two patches covert the iscsi class and drivers and fcp class.
PATCH 1/4 iscsi class, qla4xxx and libiscsi: export iscsi session state in sysfs
From: Mike Christie <[email protected]>
This patch adds some common iscsi state info to the iscsi class, so we can view it in sysfs. It also adds a functions to check the state before we queue IO (like the fc class). libiscsi does this already internally, so this is moving some libiscsi functionality to the class so hw iscsi can use it.
--- drivers/scsi/libiscsi.c | 40 +++++++++---- drivers/scsi/qla4xxx/ql4_os.c | 13 ++++ drivers/scsi/scsi_transport_iscsi.c | 111 ++++++++++++++++++++++++++++++++++- include/scsi/libiscsi.h | 18 ++++++ include/scsi/scsi_transport_iscsi.h | 27 ++++----- 5 files changed, 180 insertions(+), 29 deletions(-)
drivers/scsi/libiscsi.c +27 -13
@@ -740,6 +740,7 @@ enum {FAILURE_SESSION_TERMINATE ,FAILURE_SESSION_IN_RECOVERY ,FAILURE_SESSION_RECOVERY_TIMEOUT , + FAILURE_SESSION_NOT_READY, };int iscsi_queuecommand (struct scsi_cmnd *sc,void (*done)(struct scsi_cmnd *))
@@ -757,8 +758,14 @@ int iscsi_queuecommand(struct scsi_cmnd host = sc->device->host; session =iscsi_hostdata (host->hostdata); -spin_lock (&session->lock); + reason =iscsi_session_chkready (session_to_cls (session)); +if (reason) { + sc->result = reason; + reason =FAILURE_SESSION_NOT_READY ; + goto fault; + } +spin_lock (&session->lock);/* * ISCSI_STATE_FAILED is a temp. state. The recovery * code will decide what is best to do with command queued
@@ -777,12 +784,16 @@ int iscsi_queuecommand(struct scsi_cmnd goto reject; } -if (session->state ==ISCSI_STATE_RECOVERY_FAILED ) +if (session->state ==ISCSI_STATE_RECOVERY_FAILED ) { reason =FAILURE_SESSION_RECOVERY_TIMEOUT ; -else if (session->state ==ISCSI_STATE_TERMINATE ) + sc->result =DID_IMM_RETRY <<16 ; + }else if (session->state ==ISCSI_STATE_TERMINATE ) { reason =FAILURE_SESSION_TERMINATE ; -else + sc->result =DID_NO_CONNECT <<16 ; + }else { reason =FAILURE_SESSION_FREED ; + sc->result =DID_NO_CONNECT <<16 ; + } goto fault; }
@@ -797,6 +808,7 @@ int iscsi_queuecommand(struct scsi_cmnd conn = session->leadconn;if (!conn) { reason =FAILURE_SESSION_FREED ; + sc->result =DID_NO_CONNECT <<16 ; goto fault; }
@@ -840,7 +852,6 @@ fault:spin_unlock (&session->lock);printk (KERN_ERR "iscsi: cmd 0x%x is not queued (%d)\n" , sc->cmnd[0 ], reason); - sc->result = (DID_NO_CONNECT <<16 ); sc->resid = sc->request_bufflen; sc->scsi_done (sc);return 0 ;
@@ -1425,6 +1436,7 @@ iscsi_session_setup(struct iscsi_transpo cls_session =iscsi_create_session (shost, iscsit,0 );if (!cls_session) goto module_put; + *(unsigned long *)shost->hostdata = (unsigned long )cls_session;return cls_session;
@@ -1638,6 +1650,7 @@ int iscsi_conn_start(struct iscsi_cls_cospin_lock_bh (&session->lock); conn->c_stage =ISCSI_CONN_STARTED ; session->state =ISCSI_STATE_LOGGED_IN ; +iscsi_set_session_logged_in (session_to_cls (session));switch (conn->stop_stage) {case STOP_CONN_RECOVER :
@@ -1695,22 +1708,22 @@ flush_control_queues(struct iscsi_sessio }/* Fail commands. Mutex and session lock held and recv side suspended */ -static void fail_all_commands (struct iscsi_conn *conn) +static void fail_all_commands (struct iscsi_conn *conn,int error) {struct iscsi_cmd_task *ctask, *tmp;/* flush pending */ list_for_each_entry_safe (ctask, tmp, &conn->xmitqueue, running) { -debug_scsi ("failing pending sc %p itt 0x%x\n" , ctask->sc, - ctask->itt); -fail_command (conn, ctask,DID_BUS_BUSY <<16 ); +debug_scsi ("failing pending sc %p itt 0x%x err %d\n" , ctask->sc, + ctask->itt, error); +fail_command (conn, ctask, error <<16 ); }/* fail all other running */ list_for_each_entry_safe (ctask, tmp, &conn->run_list, running) { -debug_scsi ("failing in progress sc %p itt 0x%x\n" , - ctask->sc, ctask->itt); -fail_command (conn, ctask,DID_BUS_BUSY <<16 ); +debug_scsi ("failing in progress sc %p itt 0x%x err %d\n" , + ctask->sc, ctask->itt, error); +fail_command (conn, ctask, error <<16 ); } conn->ctask =NULL ;
@@ -1768,7 +1781,8 @@ static void iscsi_start_session_recovery* flush queues. */ spin_lock_bh (&session->lock); -fail_all_commands (conn); +fail_all_commands (conn, +STOP_CONN_RECOVER ?DID_BUS_BUSY :DID_ERROR );flush_control_queues (session, conn);spin_unlock_bh (&session->lock);
drivers/scsi/qla4xxx/ql4_os.c +13 -0
@@ -283,6 +283,7 @@ int qla4xxx_add_sess(struct ddb_entry *d } ddb_entry->sess->recovery_tmo = ddb_entry->ha->port_down_retry_count; +iscsi_set_session_logged_in (ddb_entry->sess);iscsi_if_create_session_done (ddb_entry->conn);return 0 ; }
@@ -413,9 +414,21 @@ static int qla4xxx_queuecommand(struct s {struct scsi_qla_host *ha =to_qla_host (cmd->device->host);struct ddb_entry *ddb_entry = cmd->device->hostdata; +struct iscsi_cls_session *sess = ddb_entry->sess;struct srb *srb;int rval; +if (!sess) { + cmd->result =DID_IMM_RETRY <<16 ; + goto qc_fail_command; + } + + rval =iscsi_session_chkready (sess); +if (rval) { + cmd->result = rval; + goto qc_fail_command; + } +if (atomic_read (&ddb_entry->state) !=DDB_STATE_ONLINE ) {if (atomic_read (&ddb_entry->state) ==DDB_STATE_DEAD ) { cmd->result =DID_NO_CONNECT <<16 ;
drivers/scsi/scsi_transport_iscsi.c +109 -2
189 lines not on this page · raw message
include/scsi/libiscsi.h +18 -0
@@ -119,6 +119,14 @@ struct iscsi_cmd_task {void *dd_data;/* driver/transport data */ }; +/* Connection's states */ +enum { +ISCSI_CONN_INITIAL_STAGE , +ISCSI_CONN_STARTED , +ISCSI_CONN_STOPPED , +ISCSI_CONN_CLEANUP_WAIT , +}; +struct iscsi_conn {struct iscsi_cls_conn *cls_conn;/* ptr to class connection */ void *dd_data;/* iscsi_transport data */
@@ -205,6 +213,16 @@ struct iscsi_queue {int max;/* Max number of elements */ }; +/* Session's states */ +enum { +ISCSI_STATE_FREE =1 , +ISCSI_STATE_LOGGED_IN , +ISCSI_STATE_FAILED , +ISCSI_STATE_TERMINATE , +ISCSI_STATE_IN_RECOVERY , +ISCSI_STATE_RECOVERY_FAILED , +}; +struct iscsi_session {/* iSCSI session-wide sequencing */ uint32_t cmdsn;
include/scsi/scsi_transport_iscsi.h +13 -14
@@ -141,13 +141,6 @@ extern void iscsi_conn_error(struct iscsextern int iscsi_recv_pdu (struct iscsi_cls_conn *conn,struct iscsi_hdr *hdr,char *data,uint32_t data_size); - -/* Connection's states */ -#define ISCSI_CONN_INITIAL_STAGE 0 -#define ISCSI_CONN_STARTED 1 -#define ISCSI_CONN_STOPPED 2 -#define ISCSI_CONN_CLEANUP_WAIT 3 -struct iscsi_cls_conn {struct list_head conn_list;/* item in connlist */ void *dd_data;/* LLD private data */
@@ -161,18 +154,21 @@ struct iscsi_cls_conn {#define iscsi_dev_to_conn (_dev) \ container_of(_dev, struct iscsi_cls_conn, dev) -/* Session's states */ -#define ISCSI_STATE_FREE 1 -#define ISCSI_STATE_LOGGED_IN 2 -#define ISCSI_STATE_FAILED 3 -#define ISCSI_STATE_TERMINATE 4 -#define ISCSI_STATE_IN_RECOVERY 5 -#define ISCSI_STATE_RECOVERY_FAILED 6 +#define iscsi_conn_to_session (_conn) \ + iscsi_dev_to_session(_conn->dev.parent) + +/* iscsi class session state */ +enum { +ISCSI_SESSION_LOGGED_IN , +ISCSI_SESSION_FAILED , +ISCSI_SESSION_FREE , +};struct iscsi_cls_session {struct list_head sess_list;/* item in session_list */ struct list_head host_list;struct iscsi_transport *transport; +spinlock_t lock;/* protects state */ /* recovery fields */ int recovery_tmo;
@@ -180,6 +176,7 @@ struct iscsi_cls_session {int target_id; +int state;int sid;/* session id */ void *dd_data;/* LLD private data */ struct device dev;/* sysfs transport/container device */
@@ -202,6 +199,8 @@ struct iscsi_host {/* * session and connection functions that can be used by HW iSCSI LLDs */ +extern void iscsi_set_session_logged_in (struct iscsi_cls_session *session); +extern int iscsi_session_chkready (struct iscsi_cls_session *session);extern struct iscsi_cls_session *iscsi_alloc_session (struct Scsi_Host *shost,struct iscsi_transport *transport); extern int iscsi_add_session(struct iscsi_cls_session *session,
Signature · 1 line
1.4.1.1
PATCH 2/4 scsi: add transport host byte errors
From: Mike Christie <[email protected]>
Small guide to differences in iscsi and fc:
I_T nexus or port we are connected to at some other end point
iscsi: iscsi_session
fc: rport
fast_io_fail_tmo
iscsi: session recovery_tmo
fc: rport fast_io_fail_tmo
The difference is that when the timer fires, for iscsi we unblock the queue and fail commands in the blocked queue. FC just fails IO running in the driver/fw/hw. The IO in the blocked queue sits there until dev_loss_tmo.
dev_loss_tmo
iscsi: none yet (we are working on it :))
fc: dev_loss_tmo
Currently, if there is a transport problem the iscsi drivers will return outstanding commands (commands being exeucted by the driver/fw/hw) with DID_BUS_BUSY and block the session so no new commands can be queued. Commands that are caught between the failure handling and blocking are failed with DID_IMM_RETRY or one of the scsi ml queuecommand return values. When the recovery_timeout fires, the iscsi drivers then fail IO with DID_NO_CONNECT.
For fcp, some drivers will fail some outstanding IO (disk but possibly not tape) with DID_BUS_BUSY or some other value that causes a retry and hits the scsi_error.c failfast check, block the rport, and commands caught in the race are failed with DID_IMM_RETRY. Other drivers, will hold onto all IO and wait for the terminate_rport_io or dev_loss_tmo_callbk to be called. In this case lpfc, could return the IO with DID_ERROR.
The following patches attempt unify what upper layers will see drivers like multipath can make a good guess. This relies on drivers being hooked into their transport class and implementing the terminate_rport_io callback.
This first patch just defines two new host byte errors so drivers can return the same value for when a rport/session is blocked and for when the fast_io_fail_tmo fires.
The idea is that if the LLD/class detects a problem and is going to block a rport/session, then if the LLD wants or must return the command to scsi-ml, then it can return it with DID_TRANSPORT_BLOCKED. This will requeue the IO into the same scsi queue it came from.
When using multipath and the fast_io_fail_tmo fires then the class can fail commands with DID_TRANSPORT_FAILFAST or drivers can use DID_TRANSPORT_FAILFAST in their terminate_rport_io callbacks or the equivlent in iscsi if we ever implement more advanced recovery methods. A LLD, like lpfc, could continue to return DID_ERROR and then it will hit the normal failfast path. The point of the patches is that upper layers will not see a failure that could be recovered from while the rport/session is blocked until fast_io_fail_tmo/recovery_timeout fires.
--- drivers/scsi/constants.c | 3 ++- drivers/scsi/scsi_error.c | 15 ++++++++++++++- include/scsi/scsi.h | 2 ++ 3 files changed, 18 insertions(+), 2 deletions(-)
drivers/scsi/constants.c +2 -1
@@ -1355,7 +1355,8 @@ #ifdef CONFIG_SCSI_CONSTANTSstatic const char *const hostbyte_table[]={"DID_OK" ,"DID_NO_CONNECT" ,"DID_BUS_BUSY" ,"DID_TIME_OUT" ,"DID_BAD_TARGET" ,"DID_ABORT" ,"DID_PARITY" ,"DID_ERROR" ,"DID_RESET" ,"DID_BAD_INTR" , -"DID_PASSTHROUGH" ,"DID_SOFT_ERROR" ,"DID_IMM_RETRY" }; +"DID_PASSTHROUGH" ,"DID_SOFT_ERROR" ,"DID_IMM_RETRY" ,"DID_TRANSPORT_BLOCKED" , +"DID_TRANSPORT_FAILFAST" };#define NUM_HOSTBYTE_STRS ARRAY_SIZE(hostbyte_table)static const char *const driverbyte_table[]={
drivers/scsi/scsi_error.c +14 -1
@@ -1231,7 +1231,20 @@ int scsi_decide_disposition(struct scsi_case DID_REQUEUE :return ADD_TO_MLQUEUE ; - +case DID_TRANSPORT_BLOCKED : +/* +* LLD/transport was disrupted during processing the IO. +* The transport is now blocked and attempting to recover, +* and the transport will decide what to do with the IO +* based on its timers and recovery capablilities. +*/ +return NEEDS_RETRY ; +case DID_TRANSPORT_FAILFAST : +/* +* The transport decided to failfast the IO (most likely +* the fast io fail tmo fired), so send IO directly upwards. +*/ +return SUCCESS ;case DID_ERROR :if (msg_byte (scmd->result) ==COMMAND_COMPLETE &&status_byte (scmd->result) ==RESERVATION_CONFLICT )
include/scsi/scsi.h +2 -0
@@ -309,6 +309,8 @@ #define DID_SOFT_ERROR 0x0b /* The low#define DID_IMM_RETRY 0x0c/* Retry without decrementing retry count */ #define DID_REQUEUE 0x0d/* Requeue command (no immediate retry) also * without decrementing the retry count */ +#define DID_TRANSPORT_BLOCKED 0x0e/* Transport class will handle io */ +#define DID_TRANSPORT_FAILFAST 0x0f/* Transport class fastfailed the io */ #define DRIVER_OK 0x00/* Driver status */ /*
Signature · 1 line
1.4.1.1
PATCH 3/4 iscsi class, libiscsi and qla4xxx: convert to new transport host byte values
From: Mike Christie <[email protected]>
This patch converts the iscsi drivers to the new host byte values
--- drivers/scsi/libiscsi.c | 21 ++++++++++++--------- drivers/scsi/qla4xxx/ql4_isr.c | 2 +- drivers/scsi/scsi_transport_iscsi.c | 4 ++-- 3 files changed, 15 insertions(+), 12 deletions(-)
drivers/scsi/libiscsi.c +12 -9
@@ -779,20 +779,23 @@ int iscsi_queuecommand(struct scsi_cmnd* be entering our queuecommand while a block is starting * up because the block code is not locked) */ -if (session->state ==ISCSI_STATE_IN_RECOVERY ) { +switch (session->state) { +case ISCSI_STATE_IN_RECOVERY : reason =FAILURE_SESSION_IN_RECOVERY ; - goto reject; - } - -if (session->state ==ISCSI_STATE_RECOVERY_FAILED ) { + sc->result =DID_TRANSPORT_BLOCKED <<16 ; +break ; +case ISCSI_STATE_RECOVERY_FAILED : reason =FAILURE_SESSION_RECOVERY_TIMEOUT ; - sc->result =DID_IMM_RETRY <<16 ; - }else if (session->state ==ISCSI_STATE_TERMINATE ) { + sc->result =DID_TRANSPORT_FAILFAST <<16 ; +break ; +case ISCSI_STATE_TERMINATE : reason =FAILURE_SESSION_TERMINATE ; sc->result =DID_NO_CONNECT <<16 ; - }else { +break ; +default : reason =FAILURE_SESSION_FREED ; sc->result =DID_NO_CONNECT <<16 ; +break ; } goto fault; }
@@ -1782,7 +1785,7 @@ static void iscsi_start_session_recovery*/ spin_lock_bh (&session->lock);fail_all_commands (conn, -STOP_CONN_RECOVER ?DID_BUS_BUSY :DID_ERROR ); +STOP_CONN_RECOVER ?DID_TRANSPORT_FAILFAST :DID_ERROR );flush_control_queues (session, conn);spin_unlock_bh (&session->lock);
drivers/scsi/qla4xxx/ql4_isr.c +1 -1
@@ -267,7 +267,7 @@ static void qla4xxx_status_entry(structif (atomic_read (&ddb_entry->state) ==DDB_STATE_ONLINE )qla4xxx_mark_device_missing (ha, ddb_entry); - cmd->result =DID_BUS_BUSY <<16 ; + cmd->result =DID_BLOCKED_TRANSPORT <<16 ;break ;case SCS_QUEUE_FULL :
drivers/scsi/scsi_transport_iscsi.c +2 -2
@@ -233,10 +233,10 @@ int iscsi_session_chkready(struct iscsi_ err =0 ;break ;case ISCSI_SESSION_FAILED : - err =DID_IMM_RETRY <<16 ; + err =DID_TRANSPORT_BLOCKED <<16 ;break ;case ISCSI_SESSION_FREE : - err =DID_NO_CONNECT <<16 ; + err =DID_TRANSPORT_FAILFAST <<16 ;break ;default : err =DID_NO_CONNECT <<16 ;
Signature · 1 line
1.4.1.1
PATCH 4/4 fc class: use transport host byte values
From: Mike Christie <[email protected]>
This patch is just an example. It is not tested and it is probably better to not use the fast_io_fail_tmo variable and instead modify the state machine.
This patch also does not convert any FC drivers to use the new codes.
The patch does modify the port ready function to return the new host byte values and modifies the rport fast io fail tmo to do the same thing as iscsi (fail queued and running IO).
--- drivers/scsi/scsi_transport_fc.c | 4 ++++ include/scsi/scsi_transport_fc.h | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-)
drivers/scsi/scsi_transport_fc.c +4 -0
@@ -1970,6 +1970,7 @@ fc_remote_port_add(struct Scsi_Host *sho sizeof(rport->port_name)); rport->port_id = ids->port_id; + rport->fast_io_fail_timeout =0 ; rport->port_state =FC_PORTSTATE_ONLINE ; rport->roles = ids->roles;
@@ -2059,6 +2060,7 @@ fc_remote_port_add(struct Scsi_Host *sho sizeof(rport->port_name)); rport->port_id = ids->port_id; rport->roles = ids->roles; + rport->fast_io_fail_timeout =0 ; rport->port_state =FC_PORTSTATE_ONLINE ;if (fci->f->dd_fcrport_size)
@@ -2381,7 +2383,9 @@ fc_timeout_fail_rport_io(struct work_strif (rport->port_state !=FC_PORTSTATE_BLOCKED )return ; + rport->fast_io_fail_timeout =1 ; i->f->terminate_rport_io (rport); +scsi_target_unblock (&rport->dev); }/**
include/scsi/scsi_transport_fc.h +6 -2
@@ -196,6 +196,7 @@ struct fc_rport { /* aka fc_starget_attrenum fc_port_state port_state;/* Will only be ONLINE or UNKNOWN */ u32 scsi_target_id;u32 fast_io_fail_tmo; +u8 fast_io_fail_timeout;/* exported data */ void *dd_data;/* Used for driver-specific storage */
@@ -508,17 +509,20 @@ fc_remote_port_chkready(struct fc_rport {int result; +if (rport->fastfail_timeout) +return DID_TRANSPORT_FAILFAST ; +switch (rport->port_state) {case FC_PORTSTATE_ONLINE :if (rport->roles &FC_RPORT_ROLE_FCP_TARGET ) result =0 ;else if (rport->flags &FC_RPORT_DEVLOSS_PENDING ) - result =DID_IMM_RETRY <<16 ; + result =DID_TRANSPORT_BLOCKED <<16 ;else result =DID_NO_CONNECT <<16 ;break ;case FC_PORTSTATE_BLOCKED : - result =DID_IMM_RETRY <<16 ; + result =DID_TRANSPORT_BLOCKED <<16 ;break ;default : result =DID_NO_CONNECT <<16 ;
Signature · 1 line
1.4.1.1
Re: [PATCH 2/4] scsi: add transport host byte errors
In reply to [email protected]Quoted 2 lines · [email protected] wrote:
The following patches attempt unify what upper layers will see drivers like multipath can make a good guess. This relies on drivers being
Oops, I mean to say with the patches upper layers like multipath will see common behavior from LLDs hooked into transport classes. If the class/LLD can block a rport/session and recover within fast_io_fail_tmo/replacement_timeout seconds then multipath would not see the problem. We will not have some drivers returning errors upwards and some not and some drivers holding onto errors for until the device is removed.
Re: [PATCH 2/4] scsi: add transport host byte errors
In reply to [email protected]Quoted 2 lines · [email protected] wrote:
When using multipath and the
fast_io_fail_tmofires then the class can fail commands withDID_TRANSPORT_FAILFASTor drivers can use
Bah, that is going to be wrong or ugly for tape commands.
We would want to add another new value, DID_TRANSPORT_ABORTED which would check the retries or device type. Drivers would use this when failing IO that was possibly running on the device.
Quoted 18 lines
drivers/scsi/scsi_error.c +14
-1- +case DID_TRANSPORT_BLOCKED : +/* +* LLD/transport was disrupted during processing the IO. +* The transport is now blocked and attempting to recover, +* and the transport will decide what to do with the IO +* based on its timers and recovery capablilities. +*/ +return NEEDS_RETRY ; +case DID_TRANSPORT_FAILFAST : +/* +* The transport decided to failfast the IO (most likely +* the fast io fail tmo fired), so send IO directly upwards. +*/ +return SUCCESS ;case DID_ERROR :if (msg_byte (scmd->result) ==COMMAND_COMPLETE &&status_byte (scmd->result) ==RESERVATION_CONFLICT )
Re: [PATCH 4/4] fc class: use transport host byte values
Background:
The states, in the transport are:
event state
--------------------------------
n/a running
lose connectivity blocked
fastfail timeout still blocked, but with fastfail indicator
dev_loss timeout unblocked/removed
Question:
The chkready helper in the transport catches race conditions where the block is being put in place, but queuecommand is running on another cpu, thus returning DID_IMM_RETRY.
Your patch is updating chkready to look for the case when fastfail has fired, and returning DID_TRANSPORT_FASTFAIL ? Why ? The request queue would be blocked so there should be no call to queuecommand that would fall into this category - unless you are assuming that fastfail timeout is so fast that it could fall into the same race condition as blocked.
What I think is right is - DID_TRANSPORT_FASTFAIL only is returned by the drivers on the i/o they kill as part of the terminate_io function invoked by fastfail (or on the new block queue kill request I mentioned an email ago). There should be no reason to add it to chkready unless you are trying to be absolutely protective for the above race conditions. (and hey - it makes sense that if you involve the request queue to kill all io at fast fail, there won't be any io to call queuecommand for, and even if it did get through, it'll be marked for requeuing to the request queue who will note the fastfail kill has been called and can immediately give it back to the upper layers with the proper status).
-- james s
Re: [PATCH 4/4] fc class: use transport host byte values
Quoted 7 lines · James Smart
Background:
The states, in the transport are:
event state
--------------------------------
n/a running
lose connectivity blocked
fastfail timeout still blocked, but with fastfail indicator
My patch changed this one. When this times out, I unblock the queue.
Quoted 8 lines
dev_losstimeout unblocked/removedQuestion:
The chkready helper in the transport catches race conditions where the block is being put in place, but queuecommand is running on another cpu, thus returning
DID_IMM_RETRY.
Quoted 6 lines
Your patch is updating chkready to look for the case when fastfail has fired, and returning
DID_TRANSPORT_FASTFAIL? Why ? The request queue would be blocked so there should be no call to queuecommand that would fall into this category - unless you are assuming that fastfail timeout is so fast that it could fall into the same race condition as blocked.
In the patch, when the fast IO fail timer fires, I unblock the queues.
+ rport->fast_io_fail_timeout = 1;
i->f->terminate_rport_io(rport);+ scsi_target_unblock(&rport->dev);
At that time IO would be queued on drivers and would hit that test and be failed. New IO would also be failed upwards by that check.
Re: [PATCH 4/4] fc class: use transport host byte values
ah.... OK.
At first glance, I didn't like simple addition of another flag. Although simple, it grows data structures quickly. It seems more representative of a state flag, or a iftest based on the state checking (e.g. we know we're blocked and whether we have failfast pending). But this is largely stylistic. Preferences ? I can post an alternative.
-- james
Quoted 36 lines · Mike Christie
Quoted 7 lines · James Smart
Background:
The states, in the transport are:
event state
--------------------------------
n/a running
lose connectivity blocked
fastfail timeout still blocked, but with fastfail indicatorMy patch changed this one. When this times out, I unblock the queue.
Quoted 8 lines
dev_losstimeout unblocked/removedQuestion:
The chkready helper in the transport catches race conditions where the block is being put in place, but queuecommand is running on another cpu, thus returning
DID_IMM_RETRY.Quoted 6 lines
Your patch is updating chkready to look for the case when fastfail has fired, and returning
DID_TRANSPORT_FASTFAIL? Why ? The request queue would be blocked so there should be no call to queuecommand that would fall into this category - unless you are assuming that fastfail timeout is so fast that it could fall into the same race condition as blocked.In the patch, when the fast IO fail timer fires, I unblock the queues.
(quoted diff) +2
-0+ rport->fast_io_fail_timeout = 1; i->f->terminate_rport_io(rport); + scsi_target_unblock(&rport->dev);At that time IO would be queued on drivers and would hit that test and be failed. New IO would also be failed upwards by that check.
Re: [PATCH 2/4] scsi: add transport host byte errors
In reply to [email protected]Quoted 7 lines · [email protected] wrote:
fast_io_fail_tmo
iscsi: sessionrecovery_tmo
fc: rportfast_io_fail_tmoThe difference is that when the timer fires, for iscsi we unblock the queue and fail commands in the blocked queue. FC just fails IO running in the driver/fw/hw. The IO in the blocked queue sits there until
dev_loss_tmo.
True - FC contacted the LLDD to terminate i/o, who has no notion of any io that has yet to be sent to it via queuecommand(). Blocked i/o sits until dev_loss_tmo, as that is when the sdev gets torn down. Perhaps a block layer call should be created, that the FC transport can call, to terminate the blocked queue. Thoughts ?
Quoted 12 lines
dev_loss_tmo
iscsi: none yet (we are working on it :))
fc:dev_loss_tmoCurrently, if there is a transport problem the iscsi drivers will return outstanding commands (commands being exeucted by the driver/fw/hw) with
DID_BUS_BUSYand block the session so no new commands can be queued. Commands that are caught between the failure handling and blocking are failed withDID_IMM_RETRYor one of the scsi ml queuecommand return values. When therecovery_timeoutfires, the iscsi drivers then fail IO withDID_NO_CONNECT.
Quoted 6 lines
For fcp, some drivers will fail some outstanding IO (disk but possibly not tape) with
DID_BUS_BUSYor some other value that causes a retry and hits thescsi_error.cfailfast check, block the rport, and commands caught in the race are failed withDID_IMM_RETRY. Other drivers, will hold onto all IO and wait for theterminate_rport_ioordev_loss_tmo_callbkto be called. In this case lpfc, could return the IO withDID_ERROR.
Note: Variability in behavior has to be allowed as both implementations are within FC specification. Also, the "everything killed" scenario is a valid worst case behavior that can always occur. The "it's not killed immediately" scenario is an optimization towards best-case behavior (with better FC-MI-2 compliance).
Lpfc returns DID_ERROR as the io requests had been queued to the adapter, may have gone out on the wire, and may have changed media. They were terminated early based on the respective timeout. Thus, a BUSY status, which implies no media change, is deemed inappropriate. Based on the conversation, you are implying that the layer above, which asked for the fastfail may want to distinguish between an io terminated due to the fastfail timeout vs an io that failed due to a real error. Easy enough to do - we just need a new return status. And, I see, that's what the patch below does.
So far, so good....
-- james s
Re: [PATCH 2/4] scsi: add transport host byte errors
Quoted 12 lines · James Smart
Quoted 8 lines · [email protected] wrote:
fast_io_fail_tmo
iscsi: sessionrecovery_tmo
fc: rportfast_io_fail_tmoThe difference is that when the timer fires, for iscsi we unblock the queue and fail commands in the blocked queue. FC just fails IO running in the driver/fw/hw. The IO in the blocked queue sits there until
dev_loss_tmo.
Quoted 6 lines
True - FC contacted the LLDD to terminate i/o, who has no notion of any io that has yet to be sent to it via
queuecommand(). Blocked i/o sits untildev_loss_tmo, as that is when the sdev gets torn down. Perhaps a block layer call
should be created, that the FC transport can call, to terminate the blocked queue. Thoughts ?
Do we want to fail IO that was sitting in the queue _and_ all new incoming IO or just what was sitting in the queue?
The patches I sent, unplug the queues when failfast timer expires so that is where the chk ready test for failfast comes in. When failfast fires, the queue will be unplugged and we will hit the failfast test and anything coming through will be failed.
Alternatively:
1. What about making the transport check ready test standard and adding a transportt->check_ready callout which gets called before scsi_dispatch_cmd calls the queuecommand?
2. Another option could be do add some code which does it a layer higher at the scsi device level. The function would set the scsi_device state to some value that would indicate the device is not ready and wants to fail IO, then it would unplug the queue. The scsi_prep_fn would then check for that state and fail IO. Or we could just set the state to an existing value like offline and we would not have to modify and existing state checks.
3. Or we could go one layer higher than that and add and set some block layer bits. Unblock the queue and before the scsi_prep_fn is called the block layer would check the state bit and fail IO.
4. Those would work if we want to fail IO that was queued and new incoming IO. If we want to just fail IO that was queued, and queue IO new incoming IO, then the block layer could offer a function which grabs the queue lock, dequeued what was there and then fail each IO. scsi-ml would then call that function as a helper. SCSI-ml again would not see the IO here.
Re: [PATCH 2/4] scsi: add transport host byte errors
So my position changes a little now that you pointed out that you release the request queue when fastfail kicks in.
Quoted 2 lines · Mike Christie
Do we want to fail IO that was sitting in the queue _and_ all new incoming IO or just what was sitting in the queue?
I believe all i/o - so that the upper layer sees everything occurring on each state change and can choose accordingly.
Quoted 11 lines
The patches I sent, unplug the queues when failfast timer expires so that is where the chk ready test for failfast comes in. When failfast fires, the queue will be unplugged and we will hit the failfast test and anything coming through will be failed.
Alternatively:
1. What about making the transport check ready test standard and adding a
transportt->check_readycallout which gets called beforescsi_dispatch_cmdcalls the queuecommand?
I like the idea, as I hated having to add this snippet to each driver. The other place we had to use it was in slave_alloc(), so doing the same thing there would be great.
Quoted 8 lines
2. Another option could be do add some code which does it a layer higher at the scsi device level. The function would set the
scsi_devicestate to some value that would indicate the device is not ready and wants to fail IO, then it would unplug the queue. Thescsi_prep_fnwould then check for that state and fail IO. Or we could just set the state to an existing value like offline and we would not have to modify and existing state checks.
Quoted 4 lines
3. Or we could go one layer higher than that and add and set some block layer bits. Unblock the queue and before the
scsi_prep_fnis called the block layer would check the state bit and fail IO.
Yep, and likely a better decision long term (though more work) as it's storage transport agnostic. The "other layer" guys can answer better on this one. I would think we still have to keep the chkready()s as there will always be race conditions.
Quoted 6 lines
4. Those would work if we want to fail IO that was queued and new incoming IO. If we want to just fail IO that was queued, and queue IO new incoming IO, then the block layer could offer a function which grabs the queue lock, dequeued what was there and then fail each IO. scsi-ml would then call that function as a helper. SCSI-ml again would not see the IO here.
True - but this is a new and different abort routine from the one today that expects to kick in on timeout. We purposely stopped the eh handler from aborting a command while you had connectivity lost. So, it would be a bad idea to go down this path.
-- james s
Re: [PATCH 1/4] iscsi class, qla4xxx and libiscsi: export iscsi session state in sysfs
In reply to [email protected]Quoted 5 lines
drivers/scsi/libiscsi.c static void iscsi_start_session_recovery · line 1788
Quoted 9 lines
drivers/scsi/libiscsi.c +2
-1@@ -1768,7 +1781,8 @@ static void iscsi_start_session_recovery* flush queues. */ spin_lock_bh (&session->lock); -fail_all_commands (conn); +fail_all_commands (conn, +STOP_CONN_RECOVER ?DID_BUS_BUSY :DID_ERROR );flush_control_queues (session, conn);spin_unlock_bh (&session->lock);
I'm assuming you do not support error recovery level 2, right ?
Quoted 1 line
+int
iscsi_session_chkready(struct iscsi_cls_session *session)
...
Quoted 1 line
+EXPORT_SYMBOL_GPL(iscsi_session_chkready);
Any reason this isn't an inline ?
Re: [PATCH 1/4] iscsi class, qla4xxx and libiscsi: export iscsi session state in sysfs
Quoted 4 lines · James Smart
Quoted 13 lines
drivers/scsi/libiscsi.c static void iscsi_start_session_recovery · line 1788
Quoted 10 lines
drivers/scsi/libiscsi.c +2
-1@@ -1768,7 +1781,8 @@ static void iscsi_start_session_recovery* flush queues. */ spin_lock_bh (&session->lock); -fail_all_commands (conn); +fail_all_commands (conn, +STOP_CONN_RECOVER ?DID_BUS_BUSY :DID_ERROR );flush_control_queues (session, conn);spin_unlock_bh (&session->lock);I'm assuming you do not support error recovery level 2, right ?
Yeah, we only do erl0 right now.
Quoted 6 lines
Quoted 1 line
+int
iscsi_session_chkready(struct iscsi_cls_session *session)...
Quoted 1 line
+EXPORT_SYMBOL_GPL(iscsi_session_chkready);
Any reason this isn't an inline ?
No, will make it inline.