Skip to contentSkip to conversation

Handle blocked FC rports

PATCH New Hannes Reinecke 1 participant 7 messages Mar 14, 2013 Device mapper

lore
Share & tools
lore
#0 lore raw

cover Handle blocked FC rports

Occasionally, FC HBAs might lose track of the status of the attached remote ports. This will cause I/O to be stuck and the FC HBAs to invoke the error handler, which will take _LONG_ to complete (we're speaking hours here).

This patch implements a 'suspend path' mechanism in multipath for handling these situations. Key point is to have a writeable 'port_state' sysfs attribute for the remote ports. This is introduced by the patch
'scsi_transport_fc: Make 'port_state' writeable' posted to linux-scsi.

There are two ways of suspending a path, either by a path checker returning 'PATH_TIMEOUT', or by the user invoking the new CLI command 'suspend path'.
This will then set the 'port_state' attribute to 'blocked', causing the 'fast_io_fail' mechanism to trigger. This will eventually cause all outstanding I/O to be aborted and result in a quick failover.

Paths will be resumed either from the FC HBA when a new RSCN state update is recieved from the SAN, or by the user by calling the new CLI command 'resume path'.

Hannes Reinecke (6):
libmultipath: return PATH_DOWN for quiesced paths
libmultipath: Implement PATH_TIMEOUT
libmultipath: improve debugging messages in sysfs_attr_set_value()
libmultipath: Implement sysfs_attr_get_value()
libmultipath: Implement sysfs_set_fc_rport_state()
multipathd: Implement 'suspend/resume path'

 libmultipath/checkers.h     |    5 ++++
 libmultipath/checkers/tur.c |    2 +-
 libmultipath/discovery.c    |   65 ++++++++++++++++++++++++++++++++++++++++++-
 libmultipath/discovery.h    |    1 +
 libmultipath/print.c        |    4 +++
 libmultipath/sysfs.c        |   64 ++++++++++++++++++++++++++++++++++++++++--
 libmultipath/sysfs.h        |    2 ++
 multipathd/cli.c            |    2 ++
 multipathd/cli_handlers.c   |   50 +++++++++++++++++++++++++++++++++
 multipathd/cli_handlers.h   |    2 ++
 multipathd/main.c           |    7 +++++
 11 files changed, 200 insertions(+), 4 deletions(-)
Signature · 1 line
1.7.10.4
#1 lore raw

PATCH 1/6 libmultipath: return PATH_DOWN for quiesced paths

When a SCSI device is quiesced we cannot send any I/O requests to it, so we should be returning 'PATH_DOWN' here.

---
 libmultipath/discovery.c |    3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
libmultipath/discovery.c +2 -1
index 0b5fd1d..050264e 100644
@@ -798,7 +798,8 @@ path_offline (struct path * pp)
 	condlog(3, "%s: path state = %s", pp->dev, buff);
 
 	if (!strncmp(buff, "offline", 7) ||
-	    !strncmp(buff, "transport-offline", 17)) {
+	    !strncmp(buff, "transport-offline", 17) ||
+	    !strncmp(buff, "quiesce", 7)) {
 		pp->offline = 1;
 		return PATH_DOWN;
 	}
Signature · 1 line
1.7.10.4
#2 lore raw

PATCH 2/6 libmultipath: Implement PATH_TIMEOUT

In reply to Hannes Reinecke

The tur checker might run into a timeout, eg when a command is sent but the checker hasn't been able to receive a reply in time. Use a specific 'PATH_TIMEOUT' state for these cases.

---
 libmultipath/checkers.h     |    5 +++++
 libmultipath/checkers/tur.c |    2 +-
 libmultipath/discovery.c    |    2 ++
 libmultipath/print.c        |    4 ++++
 4 files changed, 12 insertions(+), 1 deletion(-)
libmultipath/checkers.h +5 -0
index 5a96165..1b6c22d 100644
@@ -46,6 +46,10 @@
  * PATH_PENDING:
  * - Use: All async checkers
  * - Description: Indicates a check IO is in flight.
+ *
+ * PATH_TIMEOUT:
+ * - Use: Only tur checker
+ * - Description: Command timed out
  */
 enum path_check_state {
 	PATH_WILD,
@@ -55,6 +59,7 @@ enum path_check_state {
 	PATH_SHAKY,
 	PATH_GHOST,
 	PATH_PENDING,
+	PATH_TIMEOUT,
 	PATH_MAX_STATE
 };
 
libmultipath/checkers/tur.c +1 -1
index 9c141aa..6f5d4d9 100644
@@ -297,7 +297,7 @@ libcheck_check (struct checker * c)
 				pthread_cancel(ct->thread);
 				ct->running = 0;
 				MSG(c, MSG_TUR_TIMEOUT);
-				tur_status = PATH_DOWN;
+				tur_status = PATH_TIMEOUT;
 				ct->state = PATH_UNCHECKED;
 			} else {
 				condlog(3, "%d:%d: tur checker not finished",
libmultipath/discovery.c +2 -0
index 050264e..58ce533 100644
@@ -1027,6 +1027,8 @@ pathinfo (struct path *pp, vector hwtable, int mask)
 			if (pp->state == PATH_UNCHECKED ||
 			    pp->state == PATH_WILD)
 				goto blank;
+			if (pp->state == PATH_TIMEOUT)
+				pp->state = PATH_DOWN;
 		} else {
 			condlog(3, "%s: path inaccessible", pp->dev);
 			pp->chkrstate = pp->state = path_state;
libmultipath/print.c +4 -0
index 0e2c680..f40a721 100644
@@ -332,6 +332,10 @@ snprint_chk_state (char * buff, size_t len, struct path * pp)
 		return snprintf(buff, len, "shaky");
 	case PATH_GHOST:
 		return snprintf(buff, len, "ghost");
+	case PATH_PENDING:
+		return snprintf(buff, len, "i/o pending");
+	case PATH_TIMEOUT:
+		return snprintf(buff, len, "i/o timeout");
 	default:
 		return snprintf(buff, len, "undef");
 	}
Signature · 1 line
1.7.10.4
#3 lore raw

PATCH 3/6 libmultipath: improve debugging messages in sysfs_attr_set_value()

In reply to Hannes Reinecke

We should be printing out debugging messages for all failure cases in sysfs_attr_set_value().

---
 libmultipath/sysfs.c |    8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
libmultipath/sysfs.c +6 -2
index d33747f..da228e4 100644
@@ -58,12 +58,16 @@ ssize_t sysfs_attr_set_value(struct udev_device *dev, const char *attr_name,
 	}
 
 	/* skip directories */
-	if (S_ISDIR(statbuf.st_mode))
+	if (S_ISDIR(statbuf.st_mode)) {
+		condlog(4, "%s is a directory", devpath);
 		return 0;
+	}
 
 	/* skip non-writeable files */
-	if ((statbuf.st_mode & S_IWUSR) == 0)
+	if ((statbuf.st_mode & S_IWUSR) == 0) {
+		condlog(4, "%s is not writeable", devpath);
 		return 0;
+	}
 
 	/* write attribute value */
 	fd = open(devpath, O_WRONLY);
Signature · 1 line
1.7.10.4
#4 lore raw

PATCH 4/6 libmultipath: Implement sysfs_attr_get_value()

In reply to Hannes Reinecke

When we've modified an sysfs attribute via sysfs_attr_set_value() we cannot use libudev-provided functions to read the current value from that attribute, as libudev will return the stale cached value here.

---
 libmultipath/sysfs.c |   56 ++++++++++++++++++++++++++++++++++++++++++++++++++
 libmultipath/sysfs.h |    2 ++
 2 files changed, 58 insertions(+)
libmultipath/sysfs.c +56 -0
index da228e4..22b73b1 100644
@@ -38,6 +38,62 @@
 #include "debug.h"
 #include "devmapper.h"
 
+/*
+ * When we modify an attribute value we cannot rely on libudev for now,
+ * as libudev lacks the capability to update an attribute value.
+ * So for modified attributes we need to implement our own function.
+ */
+ssize_t sysfs_attr_get_value(struct udev_device *dev, const char *attr_name,
+			     char * value, size_t value_len)
+{
+	char devpath[PATH_SIZE];
+	struct stat statbuf;
+	int fd;
+	ssize_t size = -1;
+
+	if (!dev || !attr_name || !value)
+		return 0;
+
+	snprintf(devpath, PATH_SIZE, "%s/%s", udev_device_get_syspath(dev),
+		 attr_name);
+	condlog(4, "open '%s'", devpath);
+	if (stat(devpath, &statbuf) != 0) {
+		condlog(4, "stat '%s' failed: %s", devpath, strerror(errno));
+		return 0;
+	}
+
+	/* skip directories */
+	if (S_ISDIR(statbuf.st_mode)) {
+		condlog(4, "%s is a directory", devpath);
+		return 0;
+	}
+
+	/* skip non-writeable files */
+	if ((statbuf.st_mode & S_IRUSR) == 0) {
+		condlog(4, "%s is not readable", devpath);
+		return 0;
+	}
+
+	/* read attribute value */
+	fd = open(devpath, O_RDONLY);
+	if (fd < 0) {
+		condlog(4, "attribute '%s' can not be opened: %s",
+			devpath, strerror(errno));
+		return 0;
+	}
+	size = read(fd, value, value_len);
+	if (size < 0) {
+		condlog(4, "read from %s failed: %s", devpath, strerror(errno));
+		size = 0;
+	} else if (size == value_len) {
+		condlog(4, "overflow while reading from %s", devpath);
+		size = 0;
+	}
+
+	close(fd);
+	return size;
+}
+
 ssize_t sysfs_attr_set_value(struct udev_device *dev, const char *attr_name,
 			     char * value, size_t value_len)
 {
libmultipath/sysfs.h +2 -0
index 13d7545..34f3e00 100644
@@ -7,6 +7,8 @@
 
 ssize_t sysfs_attr_set_value(struct udev_device *dev, const char *attr_name,
 			     char * value, size_t value_len);
+ssize_t sysfs_attr_get_value(struct udev_device *dev, const char *attr_name,
+			     char * value, size_t value_len);
 int sysfs_get_size (struct path *pp, unsigned long long * size);
 int sysfs_check_holders(char * check_devt, char * new_devt);
 #endif
Signature · 1 line
1.7.10.4
#5 lore raw

PATCH 5/6 libmultipath: Implement sysfs_set_fc_rport_state()

In reply to Hannes Reinecke

In a faulty fabric FC HBAs might not be able to update the status of the remote ports, resulting in I/O to be stuck as the normal 'fast_io_fail' mechanism it never invoked.

To handle these cases I've implemented a function sysfs_set_fc_rport_state() which allows multipath to forcibly set the rport state to 'blocked', causing the fast_io_fail mechanism to be invoked, which eventually will abort all stuck I/O.

This patch relies on the patch
scsi_transport_fc: Make 'port_state' writeable
posted to linux-scsi.

---
 libmultipath/discovery.c |   60 ++++++++++++++++++++++++++++++++++++++++++++++
 libmultipath/discovery.h |    1 +
 2 files changed, 61 insertions(+)
libmultipath/discovery.c +60 -0
index 58ce533..3b60125 100644
@@ -476,6 +476,66 @@ sysfs_set_scsi_tmo (struct multipath *mpp)
 }
 
 int
+sysfs_set_fc_rport_state (struct path *pp, int blocked)
+{
+	struct udev_device *rport_dev = NULL;
+	char rport_id[32];
+	char value[NAME_SIZE];
+	int retval = 0;
+
+	if (pp->bus != SYSFS_BUS_SCSI) {
+		condlog(4, "%s: no FC settings on non-SCSI device", pp->dev);
+		return 0;
+	}
+
+	sprintf(rport_id, "rport-%d:%d-%d",
+		pp->sg_id.host_no, pp->sg_id.channel, pp->sg_id.transport_id);
+	rport_dev = udev_device_new_from_subsystem_sysname(conf->udev,
+				"fc_remote_ports", rport_id);
+	if (!rport_dev) {
+		condlog(1, "%s: No fc_remote_port device for '%s'", pp->dev,
+			rport_id);
+		return 0;
+	}
+
+	if (sysfs_attr_get_value(rport_dev, "port_state",
+				 value, 16) <= 0) {
+		condlog(1, "%s: failed to read rport state from '%s'",
+			pp->dev, rport_id);
+		retval = EBADF;
+		goto out;
+	}
+	condlog(3, "%s: rport state '%s'", pp->dev, value);
+	if (blocked) {
+		/* Can only transition Online -> Blocked */
+		if (strncmp(value, "Online", 6)) {
+			retval = EAGAIN;
+			goto out;
+		}
+		sprintf(value, "Blocked");
+	} else {
+		/* Can only transition Blocked -> Online */
+		if (strncmp(value, "Blocked", 7)) {
+			retval = EAGAIN;
+			goto out;
+		}
+
+		sprintf(value, "Online");
+	}
+
+	if (sysfs_attr_set_value(rport_dev, "port_state",
+				 value, strlen(value)) <= 0) {
+		condlog(1, "%s: failed to set rport to '%s', error %d",
+			pp->dev, value, errno);
+		retval = ENXIO;
+	}
+out:
+	udev_device_unref(rport_dev);
+
+	return retval;
+}
+
+int
 do_inq(int sg_fd, int cmddt, int evpd, unsigned int pg_op,
        void *resp, int mx_resp_len)
 {
libmultipath/discovery.h +1 -0
index 1a614ee..38b61dc 100644
@@ -38,6 +38,7 @@ int store_pathinfo (vector pathvec, vector hwtable,
 		    struct path **pp_ptr);
 int sysfs_set_scsi_tmo (struct multipath *mpp);
 int sysfs_get_timeout(struct path *pp, unsigned int *timeout);
+int sysfs_set_fc_rport_state(struct path *pp, int blocked);
 
 /*
  * discovery bitmask
Signature · 1 line
1.7.10.4
#6 lore raw

PATCH 6/6 multipathd: Implement 'suspend/resume path'

In reply to Hannes Reinecke

This patch implements a new CLI command pair 'suspend path' and 'resume path' to block and unblock a remote FC port manually.
It also will block the remote FC port if a path checker returns 'PATH_TIMEOUT'.

---
 multipathd/cli.c          |    2 ++
 multipathd/cli_handlers.c |   50 +++++++++++++++++++++++++++++++++++++++++++++
 multipathd/cli_handlers.h |    2 ++
 multipathd/main.c         |    7 +++++++
 4 files changed, 61 insertions(+)
multipathd/cli.c +2 -0
index d95cba0..e278956 100644
@@ -439,6 +439,8 @@ cli_init (void) {
 	add_handler(LIST+WILDCARDS, NULL);
 	add_handler(ADD+PATH, NULL);
 	add_handler(DEL+PATH, NULL);
+	add_handler(SUSPEND+PATH, NULL);
+	add_handler(RESUME+PATH, NULL);
 	add_handler(ADD+MAP, NULL);
 	add_handler(DEL+MAP, NULL);
 	add_handler(SWITCH+MAP+GROUP, NULL);
multipathd/cli_handlers.c +50 -0

56 lines not on this page · raw message

multipathd/cli_handlers.h +2 -0
index c62a273..8734610 100644
@@ -14,6 +14,8 @@ int cli_list_devices (void * v, char ** reply, int * len, void * data);
 int cli_list_wildcards (void * v, char ** reply, int * len, void * data);
 int cli_add_path (void * v, char ** reply, int * len, void * data);
 int cli_del_path (void * v, char ** reply, int * len, void * data);
+int cli_block_path (void * v, char ** reply, int * len, void * data);
+int cli_unblock_path (void * v, char ** reply, int * len, void * data);
 int cli_add_map (void * v, char ** reply, int * len, void * data);
 int cli_del_map (void * v, char ** reply, int * len, void * data);
 int cli_switch_group(void * v, char ** reply, int * len, void * data);
multipathd/main.c +7 -0
index 95264fc..30afb24 100644
@@ -861,6 +861,8 @@ uxlsnrloop (void * ap)
 	set_handler_callback(LIST+WILDCARDS, cli_list_wildcards);
 	set_handler_callback(ADD+PATH, cli_add_path);
 	set_handler_callback(DEL+PATH, cli_del_path);
+	set_handler_callback(SUSPEND+PATH, cli_block_path);
+	set_handler_callback(RESUME+PATH, cli_unblock_path);
 	set_handler_callback(ADD+MAP, cli_add_map);
 	set_handler_callback(DEL+MAP, cli_del_map);
 	set_handler_callback(SWITCH+MAP+GROUP, cli_switch_group);
@@ -1132,6 +1134,11 @@ check_path (struct vectors * vecs, struct path * pp)
 		pp->tick = 1;
 		return;
 	}
+	if (newstate == PATH_TIMEOUT) {
+		/* Command timeout, block path */
+		sysfs_set_fc_rport_state(pp, 1);
+		newstate = PATH_DOWN;
+	}
 	/*
 	 * Synchronize with kernel state
 	 */
Signature · 1 line
1.7.10.4