Handle blocked FC rports
cover Handle blocked FC rports
Occasionally, FC HBAs might lose track of the status of the attached remote ports. This will cause I/O to be stuck and the FC HBAs to invoke the error handler, which will take _LONG_ to complete (we're speaking hours here).
This patch implements a 'suspend path' mechanism in multipath for handling these situations. Key point is to have a writeable 'port_state' sysfs attribute for the remote ports. This is introduced by the patch
'scsi_transport_fc: Make 'port_state' writeable' posted to linux-scsi.
There are two ways of suspending a path, either by a path checker returning 'PATH_TIMEOUT', or by the user invoking the new CLI command 'suspend path'.
This will then set the 'port_state' attribute to 'blocked', causing the 'fast_io_fail' mechanism to trigger. This will eventually cause all outstanding I/O to be aborted and result in a quick failover.
Paths will be resumed either from the FC HBA when a new RSCN state update is recieved from the SAN, or by the user by calling the new CLI command 'resume path'.
Hannes Reinecke (6):
libmultipath: return PATH_DOWN for quiesced paths
libmultipath: Implement PATH_TIMEOUT
libmultipath: improve debugging messages in sysfs_attr_set_value()
libmultipath: Implement sysfs_attr_get_value()
libmultipath: Implement sysfs_set_fc_rport_state()
multipathd: Implement 'suspend/resume path'
libmultipath/checkers.h | 5 ++++ libmultipath/checkers/tur.c | 2 +- libmultipath/discovery.c | 65 ++++++++++++++++++++++++++++++++++++++++++- libmultipath/discovery.h | 1 + libmultipath/print.c | 4 +++ libmultipath/sysfs.c | 64 ++++++++++++++++++++++++++++++++++++++++-- libmultipath/sysfs.h | 2 ++ multipathd/cli.c | 2 ++ multipathd/cli_handlers.c | 50 +++++++++++++++++++++++++++++++++ multipathd/cli_handlers.h | 2 ++ multipathd/main.c | 7 +++++ 11 files changed, 200 insertions(+), 4 deletions(-)
Signature · 1 line
1.7.10.4
PATCH 1/6 libmultipath: return PATH_DOWN for quiesced paths
When a SCSI device is quiesced we cannot send any I/O requests to it, so we should be returning 'PATH_DOWN' here.
--- libmultipath/discovery.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
libmultipath/discovery.c +2 -1
@@ -798,7 +798,8 @@ path_offline (struct path * pp)condlog (3 ,"%s: path state = %s" , pp->dev, buff);if (!strncmp (buff,"offline" ,7 ) || - !strncmp (buff,"transport-offline" ,17 )) { + !strncmp (buff,"transport-offline" ,17 ) || + !strncmp (buff,"quiesce" ,7 )) { pp->offline =1 ;return PATH_DOWN ; }
Signature · 1 line
1.7.10.4
PATCH 2/6 libmultipath: Implement PATH_TIMEOUT
In reply to Hannes ReineckeThe tur checker might run into a timeout, eg when a command is sent but the checker hasn't been able to receive a reply in time. Use a specific 'PATH_TIMEOUT' state for these cases.
--- libmultipath/checkers.h | 5 +++++ libmultipath/checkers/tur.c | 2 +- libmultipath/discovery.c | 2 ++ libmultipath/print.c | 4 ++++ 4 files changed, 12 insertions(+), 1 deletion(-)
libmultipath/checkers.h +5 -0
@@ -46,6 +46,10 @@* PATH_PENDING: * - Use: All async checkers * - Description: Indicates a check IO is in flight. +* +* PATH_TIMEOUT: +* - Use: Only tur checker +* - Description: Command timed out */ enum path_check_state {PATH_WILD ,
@@ -55,6 +59,7 @@ enum path_check_state {PATH_SHAKY ,PATH_GHOST ,PATH_PENDING , +PATH_TIMEOUT , PATH_MAX_STATE };
libmultipath/checkers/tur.c +1 -1
@@ -297,7 +297,7 @@ libcheck_check (struct checker * c)pthread_cancel (ct->thread); ct->running =0 ;MSG (c,MSG_TUR_TIMEOUT ); - tur_status =PATH_DOWN ; + tur_status =PATH_TIMEOUT ; ct->state =PATH_UNCHECKED ; }else { condlog(3, "%d:%d: tur checker not finished",
libmultipath/discovery.c +2 -0
@@ -1027,6 +1027,8 @@ pathinfo (struct path *pp, vector hwtable, int mask)if (pp->state ==PATH_UNCHECKED || pp->state ==PATH_WILD ) goto blank; +if (pp->state ==PATH_TIMEOUT ) + pp->state =PATH_DOWN ; }else {condlog (3 ,"%s: path inaccessible" , pp->dev); pp->chkrstate = pp->state = path_state;
libmultipath/print.c +4 -0
@@ -332,6 +332,10 @@ snprint_chk_state (char * buff, size_t len, struct path * pp)return snprintf (buff, len,"shaky" );case PATH_GHOST :return snprintf (buff, len,"ghost" ); +case PATH_PENDING : +return snprintf (buff, len,"i/o pending" ); +case PATH_TIMEOUT : +return snprintf (buff, len,"i/o timeout" );default :return snprintf (buff, len,"undef" ); }
Signature · 1 line
1.7.10.4
PATCH 3/6 libmultipath: improve debugging messages in sysfs_attr_set_value()
In reply to Hannes ReineckeWe should be printing out debugging messages for all failure cases in sysfs_attr_set_value().
--- libmultipath/sysfs.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
libmultipath/sysfs.c +6 -2
@@ -58,12 +58,16 @@ ssize_t sysfs_attr_set_value(struct udev_device *dev, const char *attr_name, }/* skip directories */ -if (S_ISDIR (statbuf.st_mode)) +if (S_ISDIR (statbuf.st_mode)) { +condlog (4 ,"%s is a directory" , devpath);return 0 ; + }/* skip non-writeable files */ -if ((statbuf.st_mode &S_IWUSR ) ==0 ) +if ((statbuf.st_mode &S_IWUSR ) ==0 ) { +condlog (4 ,"%s is not writeable" , devpath);return 0 ; + }/* write attribute value */ fd =open (devpath,O_WRONLY );
Signature · 1 line
1.7.10.4
PATCH 4/6 libmultipath: Implement sysfs_attr_get_value()
In reply to Hannes ReineckeWhen we've modified an sysfs attribute via sysfs_attr_set_value() we cannot use libudev-provided functions to read the current value from that attribute, as libudev will return the stale cached value here.
--- libmultipath/sysfs.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++ libmultipath/sysfs.h | 2 ++ 2 files changed, 58 insertions(+)
libmultipath/sysfs.c +56 -0
@@ -38,6 +38,62 @@#include "debug.h" #include "devmapper.h" +/* +* When we modify an attribute value we cannot rely on libudev for now, +* as libudev lacks the capability to update an attribute value. +* So for modified attributes we need to implement our own function. +*/ +ssize_t sysfs_attr_get_value (struct udev_device *dev,const char *attr_name, +char * value,size_t value_len) +{ +char devpath[PATH_SIZE ]; +struct stat statbuf; +int fd; +ssize_t size =-1 ; + +if (!dev || !attr_name || !value) +return 0 ; + +snprintf (devpath,PATH_SIZE ,"%s/%s" ,udev_device_get_syspath (dev), + attr_name); +condlog (4 ,"open '%s'" , devpath); +if (stat (devpath, &statbuf) !=0 ) { +condlog (4 ,"stat '%s' failed: %s" , devpath,strerror (errno)); +return 0 ; + } + +/* skip directories */ +if (S_ISDIR (statbuf.st_mode)) { +condlog (4 ,"%s is a directory" , devpath); +return 0 ; + } + +/* skip non-writeable files */ +if ((statbuf.st_mode &S_IRUSR ) ==0 ) { +condlog (4 ,"%s is not readable" , devpath); +return 0 ; + } + +/* read attribute value */ + fd =open (devpath,O_RDONLY ); +if (fd <0 ) { +condlog (4 ,"attribute '%s' can not be opened: %s" , + devpath,strerror (errno)); +return 0 ; + } + size =read (fd, value, value_len); +if (size <0 ) { +condlog (4 ,"read from %s failed: %s" , devpath,strerror (errno)); + size =0 ; + }else if (size == value_len) { +condlog (4 ,"overflow while reading from %s" , devpath); + size =0 ; + } + +close (fd); +return size; +} +ssize_t sysfs_attr_set_value (struct udev_device *dev,const char *attr_name,char * value,size_t value_len) {
libmultipath/sysfs.h +2 -0
@@ -7,6 +7,8 @@ssize_t sysfs_attr_set_value (struct udev_device *dev,const char *attr_name,char * value,size_t value_len); +ssize_t sysfs_attr_get_value (struct udev_device *dev,const char *attr_name, +char * value,size_t value_len);int sysfs_get_size (struct path *pp,unsigned long long * size);int sysfs_check_holders (char * check_devt,char * new_devt);#endif
Signature · 1 line
1.7.10.4
PATCH 5/6 libmultipath: Implement sysfs_set_fc_rport_state()
In reply to Hannes ReineckeIn a faulty fabric FC HBAs might not be able to update the status of the remote ports, resulting in I/O to be stuck as the normal 'fast_io_fail' mechanism it never invoked.
To handle these cases I've implemented a function sysfs_set_fc_rport_state() which allows multipath to forcibly set the rport state to 'blocked', causing the fast_io_fail mechanism to be invoked, which eventually will abort all stuck I/O.
This patch relies on the patchscsi_transport_fc: Make 'port_state' writeable
posted to linux-scsi.
--- libmultipath/discovery.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++ libmultipath/discovery.h | 1 + 2 files changed, 61 insertions(+)
libmultipath/discovery.c +60 -0
@@ -476,6 +476,66 @@ sysfs_set_scsi_tmo (struct multipath *mpp) }int +sysfs_set_fc_rport_state (struct path *pp,int blocked) +{ +struct udev_device *rport_dev =NULL ; +char rport_id[32 ]; +char value[NAME_SIZE ]; +int retval =0 ; + +if (pp->bus !=SYSFS_BUS_SCSI ) { +condlog (4 ,"%s: no FC settings on non-SCSI device" , pp->dev); +return 0 ; + } + +sprintf (rport_id,"rport-%d:%d-%d" , + pp->sg_id.host_no, pp->sg_id.channel, pp->sg_id.transport_id); + rport_dev =udev_device_new_from_subsystem_sysname (conf->udev, +"fc_remote_ports" , rport_id); +if (!rport_dev) { +condlog (1 ,"%s: No fc_remote_port device for '%s'" , pp->dev, + rport_id); +return 0 ; + } + +if (sysfs_attr_get_value (rport_dev,"port_state" , + value,16 ) <=0 ) { +condlog (1 ,"%s: failed to read rport state from '%s'" , + pp->dev, rport_id); + retval =EBADF ; + goto out; + } +condlog (3 ,"%s: rport state '%s'" , pp->dev, value); +if (blocked) { +/* Can only transition Online -> Blocked */ +if (strncmp (value,"Online" ,6 )) { + retval =EAGAIN ; + goto out; + } +sprintf (value,"Blocked" ); + }else { +/* Can only transition Blocked -> Online */ +if (strncmp (value,"Blocked" ,7 )) { + retval =EAGAIN ; + goto out; + } + +sprintf (value,"Online" ); + } + +if (sysfs_attr_set_value (rport_dev,"port_state" , + value,strlen (value)) <=0 ) { +condlog (1 ,"%s: failed to set rport to '%s', error %d" , + pp->dev, value, errno); + retval =ENXIO ; + } +out: +udev_device_unref (rport_dev); + +return retval; +} + +int do_inq (int sg_fd,int cmddt,int evpd,unsigned int pg_op,void *resp,int mx_resp_len) {
libmultipath/discovery.h +1 -0
@@ -38,6 +38,7 @@ int store_pathinfo (vector pathvec, vector hwtable, struct path **pp_ptr);int sysfs_set_scsi_tmo (struct multipath *mpp);int sysfs_get_timeout (struct path *pp,unsigned int *timeout); +int sysfs_set_fc_rport_state (struct path *pp,int blocked);/* * discovery bitmask
Signature · 1 line
1.7.10.4
PATCH 6/6 multipathd: Implement 'suspend/resume path'
In reply to Hannes ReineckeThis patch implements a new CLI command pair 'suspend path' and 'resume path' to block and unblock a remote FC port manually.
It also will block the remote FC port if a path checker returns 'PATH_TIMEOUT'.
--- multipathd/cli.c | 2 ++ multipathd/cli_handlers.c | 50 +++++++++++++++++++++++++++++++++++++++++++++ multipathd/cli_handlers.h | 2 ++ multipathd/main.c | 7 +++++++ 4 files changed, 61 insertions(+)
multipathd/cli.c +2 -0
@@ -439,6 +439,8 @@ cli_init (void) {add_handler (LIST +WILDCARDS ,NULL );add_handler (ADD +PATH ,NULL );add_handler (DEL +PATH ,NULL ); +add_handler (SUSPEND +PATH ,NULL ); +add_handler (RESUME +PATH ,NULL );add_handler (ADD +MAP ,NULL );add_handler (DEL +MAP ,NULL );add_handler (SWITCH +MAP +GROUP ,NULL );
multipathd/cli_handlers.c +50 -0
56 lines not on this page · raw message
multipathd/cli_handlers.h +2 -0
@@ -14,6 +14,8 @@ int cli_list_devices (void * v, char ** reply, int * len, void * data);int cli_list_wildcards (void * v,char ** reply,int * len,void * data);int cli_add_path (void * v,char ** reply,int * len,void * data);int cli_del_path (void * v,char ** reply,int * len,void * data); +int cli_block_path (void * v,char ** reply,int * len,void * data); +int cli_unblock_path (void * v,char ** reply,int * len,void * data);int cli_add_map (void * v,char ** reply,int * len,void * data);int cli_del_map (void * v,char ** reply,int * len,void * data);int cli_switch_group (void * v,char ** reply,int * len,void * data);
multipathd/main.c +7 -0
@@ -861,6 +861,8 @@ uxlsnrloop (void * ap)set_handler_callback (LIST +WILDCARDS , cli_list_wildcards);set_handler_callback (ADD +PATH , cli_add_path);set_handler_callback (DEL +PATH , cli_del_path); +set_handler_callback (SUSPEND +PATH , cli_block_path); +set_handler_callback (RESUME +PATH , cli_unblock_path);set_handler_callback (ADD +MAP , cli_add_map);set_handler_callback (DEL +MAP , cli_del_map);set_handler_callback (SWITCH +MAP +GROUP , cli_switch_group);
@@ -1132,6 +1134,11 @@ check_path (struct vectors * vecs, struct path * pp) pp->tick =1 ;return ; } +if (newstate ==PATH_TIMEOUT ) { +/* Command timeout, block path */ +sysfs_set_fc_rport_state (pp,1 ); + newstate =PATH_DOWN ; + }/* * Synchronize with kernel state */
Signature · 1 line
1.7.10.4