Skip to contentSkip to conversation

Re-integrate a failed secondary mirror leg

PATCH New Malahal Naineni 4 participants 17 messages Dec 13, 2009 – Dec 23, 2009

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Re-integrate a failed secondary mirror leg

The following 4 patches try to re-integrate a failed secondary mirror leg.

Patch 1: Re-arranges some code and introduces more error type returns Patch 2: Tries to re-integrate the failed device indefinitely Patch 3: Removes some "#ifdef 0' code and fixes timeout setup mechanism.
Also adds interface to remove a timeout setup
Patch 4: Uses Patch 2 and Patch 3 to attempt only few times at a regular
interval to resync the failed mirror.

Tested with flakey dm module!

#1 lore raw

PATCH 1/1 Add more error codes in mirror DSO

The mirror DSO (daemons/dmeventd/plugins/mirror/dmeventd_mirror.c) logs various device failure codes but only handles one failure, 'ME_FAILURE'. This patchs adds more error codes to handle multiple types of device failures.

daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +89 -56
diff -r a48a5f0eea85 -r a74600c6163e daemons/dmeventd/plugins/mirror/dmeventd_mirror.c
@@ -30,9 +30,17 @@
 /* FIXME Replace most syslogs with log_error() style messages and add complete context. */
 /* FIXME Reformat to 80 char lines. */
 
-#define ME_IGNORE    0
-#define ME_INSYNC    1
-#define ME_FAILURE   2
+/* 
+ * An event may contain more than one error type. The following are bit
+ * flags that indicate each error type.
+ */
+#define ME_IGNORE    			0x01U
+#define ME_INSYNC    			0x02U
+#define ME_READ_FAILURE   		0x04U
+#define ME_SYNC_FAILURE   		0x08U
+#define ME_LOG_FAILURE 			0x10U
+#define ME_SECONDARY_WRITE_FAILURE	0x20U
+#define ME_PRIMARY_WRITE_FAILURE	0x40U
 
 /*
  * register_device() is called first and performs initialisation.
@@ -53,45 +61,68 @@ static void *_lvm_handle = NULL;
  */
 static pthread_mutex_t _event_mutex = PTHREAD_MUTEX_INITIALIZER;
 
-static int _process_status_code(const char status_code, const char *dev_name,
-				const char *dev_type, int r)
+static int _process_status_code(const char *mirror, const char *device,
+			        const char status_code, int dev_index)
 {
-	/*
-	 *    A => Alive - No failures
-	 *    D => Dead - A write failure occurred leaving mirror out-of-sync
-	 *    F => Flush failed.
-	 *    S => Sync - A sychronization failure occurred, mirror out-of-sync
-	 *    R => Read - A read failure occurred, mirror data unaffected
-	 *    U => Unclassified failure (bug)
-	 */ 
-	if (status_code == 'F') {
-		syslog(LOG_ERR, "%s device %s flush failed.\n",
-		       dev_type, dev_name);
-		r = ME_FAILURE;
-	} else if (status_code == 'S')
-		syslog(LOG_ERR, "%s device %s sync failed.\n",
-		       dev_type, dev_name);
-	else if (status_code == 'R')
-		syslog(LOG_ERR, "%s device %s read failed.\n",
-		       dev_type, dev_name);
-	else if (status_code != 'A') {
-		syslog(LOG_ERR, "%s device %s has failed (%c).\n",
-		       dev_type, dev_name, status_code);
-		r = ME_FAILURE;
+	int r;
+	const char *error_string;
+	const char *dev_type = dev_index ? "Secondary" : "Primary";
+
+	switch (status_code) {
+	default:
+	case 'U': /* Unclassified failure (bug) */
+	case 'D': /* Write failure occurred leaving mirror out-of-sync */
+	case 'F': /* Flush failure, handled as write failure */
+		if (status_code == 'D')
+			error_string = "write";
+		else if (status_code == 'F')
+			error_string = "flush";
+		else if (status_code == 'U')
+			error_string = "unclassified";
+		else {
+			syslog(LOG_ERR, "Unknown device status code (%c).\n",
+					status_code);
+			error_string = "unknown";
+		}
+
+		syslog(LOG_ERR, "Mirror device: %s, %s leg: %s had a %s "
+				"failure.\n", mirror, dev_type, device,
+				error_string);
+
+		r = dev_index? ME_SECONDARY_WRITE_FAILURE :
+			ME_PRIMARY_WRITE_FAILURE;
+		break;
+
+	case 'S': /* Sychronization failure occurred, mirror out-of-sync */
+		syslog(LOG_ERR, "Mirror device: %s, %s leg: %s had "
+				"sync failure.\n", mirror, dev_type, device);
+		r = ME_SYNC_FAILURE;
+		break;
+
+	case 'R': /* Read failure occurred, mirror data unaffected */
+		syslog(LOG_ERR, "Mirror device: %s, %s leg: %s had a "
+				"read failure.\n", mirror, dev_type, device);
+		r = ME_READ_FAILURE;
+		break;
+
+	case 'A': /* Active, a good status */
+		r = 0;
+		break;
 	}
 
 	return r;
 }
 
-static int _get_mirror_event(char *params)
+static int _get_mirror_event(const char *mirror, char *params)
 {
-	int i, r = ME_INSYNC;
+	int i;
 	char **args = NULL;
 	char *dev_status_str;
 	char *log_status_str;
 	char *sync_str;
 	char *p = NULL;
 	int log_argc, num_devs;
+	int retval = 0;
 
 	/*
 	 * dm core parms:	     0 409600 mirror
@@ -122,23 +153,25 @@ static int _get_mirror_event(char *param
 
 	/* Check for bad mirror devices */
 	for (i = 0; i < num_devs; i++)
-		r = _process_status_code(dev_status_str[i], args[i],
-			i ? "Secondary mirror" : "Primary mirror", r);
+		retval |= _process_status_code(mirror, args[i],
+					       dev_status_str[i], i);
 
 	/* Check for bad disk log device */
-	if (log_argc > 1)
-		r = _process_status_code(log_status_str[0],
-					 args[2 + num_devs + log_argc],
-					 "Log", r);
+	if (log_argc > 1 && log_status_str[0] == 'D') {
+		syslog(LOG_ERR, "Mirror device: %s, log device: %s failed.\n",
+				mirror, args[2 + num_devs + log_argc]);
+		retval = retval | ME_LOG_FAILURE;
+	}
 
-	if (r == ME_FAILURE)
+	if (retval)	/* A failure occurred */
 		goto out;
 
+	retval = ME_INSYNC;	/* assume INSYNC event */
 	p = strstr(sync_str, "/");
 	if (p) {
 		p[0] = '\0';
 		if (strcmp(sync_str, p+1))
-			r = ME_IGNORE;
+			retval = ME_IGNORE;
 		p[0] = '/';
 	} else
 		goto out_parse;
@@ -146,7 +179,7 @@ static int _get_mirror_event(char *param
 out:
 	if (args)
 		dm_free(args);
-	return r;
+	return retval;
 	
 out_parse:
 	if (args)
@@ -212,6 +245,7 @@ void process_event(struct dm_task *dmt,
 	char *target_type = NULL;
 	char *params;
 	const char *device = dm_task_get_name(dmt);
+	int error;
 
 	if (pthread_mutex_trylock(&_event_mutex)) {
 		syslog(LOG_NOTICE, "Another thread is handling an event.  Waiting...");
@@ -231,17 +265,11 @@ void process_event(struct dm_task *dmt,
 			continue;
 		}
 
-		switch(_get_mirror_event(params)) {
-		case ME_INSYNC:
-			/* FIXME: all we really know is that this
-			   _part_ of the device is in sync
-			   Also, this is not an error
-			*/
-			syslog(LOG_NOTICE, "%s is now in-sync\n", device);
-			break;
-		case ME_FAILURE:
-			syslog(LOG_ERR, "Device failure in %s\n", device);
-			if (_remove_failed_devices(device))
+		error = _get_mirror_event(device, params);
+		if (error & ME_LOG_FAILURE ||
+		    error & ME_PRIMARY_WRITE_FAILURE ||
+		    error & ME_SECONDARY_WRITE_FAILURE) {
+			if (_remove_failed_devices(device)) {
 				/* FIXME Why are all the error return codes unused? Get rid of them? */
 				syslog(LOG_ERR, "Failed to remove faulty devices in %s\n",
 				       device);
@@ -250,13 +278,18 @@ void process_event(struct dm_task *dmt,
 				syslog(LOG_NOTICE, "%s is now a linear device.\n",
 					device);
 			*/
-			break;
-		case ME_IGNORE:
-			break;
-		default:
-			/* FIXME Provide value then! */
-			syslog(LOG_INFO, "Unknown event received.\n");
-		}
+			}
+		} else if (error & ME_INSYNC) {
+			/* FIXME: all we really know is that this
+			   _part_ of the device is in sync
+			   Also, this is not an error
+			*/
+			syslog(LOG_NOTICE, "%s is now in-sync\n", device);
+		} else if (error & ME_READ_FAILURE ||
+			   error & ME_SYNC_FAILURE) {
+			/* Ignore these for now */
+		} else
+			syslog(LOG_INFO, "Unknown event:%u received.\n", error);
 	} while (next);
 
 	pthread_mutex_unlock(&_event_mutex);
#5 lore raw

PATCH 1/1 Handle transient secondary mirror leg failures

In reply to Malahal Naineni

A new mirror_device_fault_policy, "retry", is added to handle transient device failures. When this policy is selected, the mirror DSO will try to resync the mirror upon a secondary leg failure. This will be tried and tried until the mirror goes to in_sync state. Later patches make this a configurable number spaced at some configurable timeout.

The patch uses dmsetup suspend and resume commands to attempt a resync. It uses "lvm dumpconfig" to find out the mirror_device_fault_policy.

Signed-off-by Malahal Naineni (malahal at us.ibm.com)
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +179 -9
diff -r a74600c6163e -r 1e369d480df0 daemons/dmeventd/plugins/mirror/dmeventd_mirror.c
@@ -14,6 +14,7 @@
 
 #include "lvm2cmd.h"
 #include "errors.h"
+#include "defaults.h"
 
 #include <libdevmapper.h>
 #include <libdevmapper-event.h>
@@ -24,6 +25,7 @@
 #include <stdlib.h>
 #include <pthread.h>
 #include <unistd.h>
+#include <ctype.h>
 
 #include <syslog.h> /* FIXME Replace syslog with multilog */
 /* FIXME Missing openlog? */
@@ -56,6 +58,109 @@ static int _register_count = 0;
 static struct dm_pool *_mem_pool = NULL;
 static void *_lvm_handle = NULL;
 
+enum fault_policy {
+	FAULT_POLICY_INVALID,
+	FAULT_POLICY_REMOVE,
+	FAULT_POLICY_ALLOCATE,
+	FAULT_POLICY_ALLOCATE_ANYWHERE,
+	FAULT_POLICY_RETRY,
+};
+
+struct mirror_device_info {
+	enum fault_policy fault_policy;
+};
+
+#define CMD_SIZE 256	/* FIXME Use system restriction */
+#define LINE_SIZE 1024	/* FIXME Use system restriction */
+static int fill_config_str(char buf[], int bufsize, const char *path)
+{
+	char cmd_str[CMD_SIZE];
+	char data[LINE_SIZE], *ptr;
+	int ret = 0;
+	FILE *fp;
+
+	snprintf(cmd_str, sizeof(cmd_str), "lvm dumpconfig %s", path);
+	if ((fp = popen(cmd_str, "r")) == NULL) {
+		syslog(LOG_ERR, "fopen() failed.\n");
+		goto out;
+	}
+
+	if ((ptr = fgets(data, sizeof(data), fp)) == NULL) {
+		syslog(LOG_ERR, "fgets() failed.\n");
+		pclose(fp);
+		goto out;
+	}
+
+	/* Remove white space or quotes at the end */
+	for (ptr = &data[strlen(data)-1]; ptr >= data; --ptr)
+		if (isspace(*ptr) || *ptr == '\"' || *ptr == '\'')
+			*ptr = '\0';
+		else
+			break;
+	ptr = strchr(data, '=');
+	if (ptr) {
+		ptr++; /* After the '=' sign */
+		/* skip quotes or white space */
+		if (isspace(*ptr) || *ptr == '\"' || *ptr == '\'')
+			ptr++;
+		strncpy(buf, ptr, bufsize);
+		buf[bufsize-1] = '\0';
+		ret = 1; /* buf is valid */
+	}
+	if (pclose(fp) != 0)
+		syslog(LOG_ERR, "pclose() failed.\n");
+
+out:
+	return ret;
+}
+
+static enum fault_policy fault_policy_str2enum(const char *str)
+{
+	enum fault_policy ret = FAULT_POLICY_INVALID;
+
+	/* At most the loop is ran twice */
+	while (ret == FAULT_POLICY_INVALID) {
+		if (!strcmp(str, "remove"))
+			ret = FAULT_POLICY_REMOVE;
+		else if (!strcmp(str, "allocate"))
+			ret = FAULT_POLICY_ALLOCATE;
+		else if (!strcmp(str, "allocate_anywhere"))
+			ret = FAULT_POLICY_ALLOCATE_ANYWHERE;
+		else if (!strcmp(str, "retry"))
+			ret = FAULT_POLICY_RETRY;
+		else {
+			syslog(LOG_ERR, "Bad activation/"
+					"mirror_device_fault_policy: %s\n",
+					str);
+			str = DEFAULT_MIRROR_DEV_FAULT_POLICY;
+		}
+	}
+
+	return ret;
+}
+
+
+static enum fault_policy get_mirror_fault_policy()
+{
+	enum fault_policy ret;
+	char policy[LINE_SIZE];
+	const char *ptr;
+
+	ret = fill_config_str(policy, sizeof(policy),
+			"activation/mirror_device_fault_policy");
+	if (ret) {
+		if (!strcmp(policy, ""))
+			ptr = DEFAULT_MIRROR_DEV_FAULT_POLICY;
+		else
+			ptr = policy;
+	} else {
+		ptr = DEFAULT_MIRROR_DEV_FAULT_POLICY;
+	}
+	ret = fault_policy_str2enum(ptr);
+
+	return ret;
+}
+
 /*
  * Currently only one event can be processed at a time.
  */
@@ -200,10 +305,52 @@ static void _temporary_log_fn(int level,
 		syslog(LOG_DEBUG, "%s", format);
 }
 
+
+static int retry_failed_devices(const char *device)
+{
+	int r;
+	char cmd_str[CMD_SIZE];
+	char *vg = NULL, *lv = NULL, *layer = NULL;
+
+	if (strlen(device) > 200)  /* FIXME Use real restriction */
+		/* FIXME These return code distinctions are not used so
+		 * remove them! */
+		return -ENAMETOOLONG;
+
+	if (!dm_split_lvm_name(_mem_pool, device, &vg, &lv, &layer)) {
+		syslog(LOG_ERR, "Unable to determine VG name from %s",
+		       device);
+		/* FIXME Replace with generic error return - reason for
+		 * failure has already got logged */
+		return -ENOMEM;
+	}
+
+	/* FIXME: should be running an LVM command that is pinned.
+	 * dmsetup command may not be pinned in memory all the time.
+	 * "lvchange --refresh vg/lv" only works if there are no device
+	 * failures while it is running. Otherwise, the failed device
+	 * is replaced with "error" target which is not what we want.
+	 */
+	snprintf(cmd_str, CMD_SIZE, "dmsetup suspend --noflush %s-%s", vg, lv);
+	syslog(LOG_NOTICE, "Running command: %s", cmd_str);
+	r = system(cmd_str);
+
+	snprintf(cmd_str, CMD_SIZE, "dmsetup table %s-%s | dmsetup load "
+			"%s-%s", vg, lv, vg, lv);
+	syslog(LOG_NOTICE, "Running command: %s", cmd_str);
+	r |= system(cmd_str);
+
+	snprintf(cmd_str, CMD_SIZE, "dmsetup resume %s-%s", vg, lv);
+	syslog(LOG_NOTICE, "Running command: %s", cmd_str);
+	r |= system(cmd_str);
+
+	dm_pool_empty(_mem_pool);  /* FIXME: not safe with multiple threads */
+	return r;
+}
+
 static int _remove_failed_devices(const char *device)
 {
 	int r;
-#define CMD_SIZE 256	/* FIXME Use system restriction */
 	char cmd_str[CMD_SIZE];
 	char *vg = NULL, *lv = NULL, *layer = NULL;
 
@@ -238,7 +385,7 @@ static int _remove_failed_devices(const 
 
 void process_event(struct dm_task *dmt,
 		   enum dm_event_mask event __attribute((unused)),
-		   void **unused __attribute((unused)))
+		   void **private)
 {
 	void *next = NULL;
 	uint64_t start, length;
@@ -246,6 +393,7 @@ void process_event(struct dm_task *dmt,
 	char *params;
 	const char *device = dm_task_get_name(dmt);
 	int error;
+	struct mirror_device_info *mirror_info = *private;
 
 	if (pthread_mutex_trylock(&_event_mutex)) {
 		syslog(LOG_NOTICE, "Another thread is handling an event.  Waiting...");
@@ -268,8 +416,17 @@ void process_event(struct dm_task *dmt,
 		error = _get_mirror_event(device, params);
 		if (error & ME_LOG_FAILURE ||
 		    error & ME_PRIMARY_WRITE_FAILURE ||
-		    error & ME_SECONDARY_WRITE_FAILURE) {
-			if (_remove_failed_devices(device)) {
+		    error & ME_SECONDARY_WRITE_FAILURE ||
+		    error & ME_SYNC_FAILURE) {
+ 			if (mirror_info->fault_policy == FAULT_POLICY_RETRY &&
+					(error & ME_SECONDARY_WRITE_FAILURE ||
+					 error & ME_SYNC_FAILURE)) {
+ 				syslog(LOG_ERR, "Retrying the failed mirror "
+ 						"device.\n");
+ 				if (retry_failed_devices(device))
+ 					syslog(LOG_ERR, "Failed to reload the "
+ 							"mirror: %s\n", device);
+ 			} else if (_remove_failed_devices(device)) {
 				/* FIXME Why are all the error return codes unused? Get rid of them? */
 				syslog(LOG_ERR, "Failed to remove faulty devices in %s\n",
 				       device);
@@ -285,9 +442,8 @@ void process_event(struct dm_task *dmt,
 			   Also, this is not an error
 			*/
 			syslog(LOG_NOTICE, "%s is now in-sync\n", device);
-		} else if (error & ME_READ_FAILURE ||
-			   error & ME_SYNC_FAILURE) {
-			/* Ignore these for now */
+		} else if (error & ME_READ_FAILURE) {
+			/* Ignore it for now */
 		} else
 			syslog(LOG_INFO, "Unknown event:%u received.\n", error);
 	} while (next);
@@ -299,9 +455,10 @@ int register_device(const char *device,
 		    const char *uuid __attribute((unused)),
 		    int major __attribute((unused)),
 		    int minor __attribute((unused)),
-		    void **unused __attribute((unused)))
+		    void **private)
 {
 	int r = 0;
+	struct mirror_device_info *mirror_info;
 
 	pthread_mutex_lock(&_register_mutex);
 
@@ -312,9 +469,19 @@ int register_device(const char *device,
 	if (!_mem_pool && !(_mem_pool = dm_pool_create("mirror_dso", 1024)))
 		goto out;
 
+	mirror_info = dm_malloc(sizeof(struct mirror_device_info));
+	if (!mirror_info) {
+		dm_pool_destroy(_mem_pool);
+		_mem_pool = NULL;
+		goto out;
+	}
+	mirror_info->fault_policy = get_mirror_fault_policy();
+	*private = mirror_info;
+
 	if (!_lvm_handle) {
 		lvm2_log_fn(_temporary_log_fn);
 		if (!(_lvm_handle = lvm2_init())) {
+			dm_free(mirror_info);
 			dm_pool_destroy(_mem_pool);
 			_mem_pool = NULL;
 			goto out;
@@ -339,8 +506,11 @@ int unregister_device(const char *device
 		      const char *uuid __attribute((unused)),
 		      int major __attribute((unused)),
 		      int minor __attribute((unused)),
-		      void **unused __attribute((unused)))
+		      void **private)
 {
+	struct mirror_device_info *mirror_info = *private;
+
+	dm_free(mirror_info);
 	pthread_mutex_lock(&_register_mutex);
 
 	syslog(LOG_INFO, "No longer monitoring mirror device %s for events\n",
doc/example.conf +3 -0
diff -r a74600c6163e -r 1e369d480df0 doc/example.conf
@@ -403,6 +403,9 @@ activation {
     #            since it would break the redundant nature of the mirror. This
     #            policy acts like "remove" if no suitable device and space can
     #            be allocated for the replacement.
+    #
+    # "retry" - Try to re-integrate the failed mirror leg assuming that the
+    #           failure is transient. Not implemented yet, so don't use it.
 
     mirror_log_fault_policy = "allocate"
     mirror_device_fault_policy = "remove"
lib/metadata/mirror.c +3 -0
diff -r a74600c6163e -r 1e369d480df0 lib/metadata/mirror.c
@@ -37,6 +37,7 @@
 #define MIRROR_REMOVE		 0
 #define MIRROR_ALLOCATE		 1
 #define MIRROR_ALLOCATE_ANYWHERE 2
+#define MIRROR_RETRY	 	 3
 
 /*
  * Returns true if the lv is temporary mirror layer for resync
@@ -787,6 +788,8 @@ static int get_mirror_fault_policy(struc
 		return MIRROR_ALLOCATE;
 	else if (!strcmp(policy, "allocate_anywhere"))
 		return MIRROR_ALLOCATE_ANYWHERE;
+	else if (!strcmp(policy, "retry"))
+		return MIRROR_RETRY;
 
 	if (log_policy)
 		log_error("Bad activation/mirror_log_fault_policy");
#12 lore raw

PATCH 1/1 Add dm_event_set_timeout/dm_event_unset_timeout interface

In reply to Malahal Naineni

Will be used in mirror DSO to handle transient device failures.

daemons/dmeventd/.exported_symbols +2 -0
diff -r 1e369d480df0 -r 1c364c686b5e daemons/dmeventd/.exported_symbols
@@ -17,3 +17,5 @@ dm_event_unregister_handler
 dm_event_get_registered_device
 dm_event_handler_set_timeout
 dm_event_handler_get_timeout
+dm_event_set_timeout
+dm_event_unset_timeout
daemons/dmeventd/dmeventd.c +15 -3
diff -r 1e369d480df0 -r 1c364c686b5e daemons/dmeventd/dmeventd.c
@@ -1164,13 +1164,25 @@ static int _get_next_registered_device(s
 static int _set_timeout(struct message_data *message_data)
 {
 	struct thread_status *thread;
+	int ret = 0;
 
 	_lock_mutex();
-	if ((thread = _lookup_thread_status(message_data)))
-		thread->timeout = message_data->timeout.secs;
+	if ((thread = _lookup_thread_status(message_data))) {
+		/* INT_MAX timeout is used to unregister timeout,
+		 * register timeout for any other value. */
+		thread->events &= ~DM_EVENT_TIMEOUT;
+		_unregister_for_timeout(thread);
+		if (message_data->timeout.secs == INT_MAX) {
+			thread->timeout = 0;
+		} else {
+			thread->events |= DM_EVENT_TIMEOUT;
+			thread->timeout = message_data->timeout.secs;
+			ret = _register_for_timeout(thread);
+		}
+	}
 	_unlock_mutex();
 
-	return thread ? 0 : -ENODEV;
+	return (thread && ret == 0) ? 0 : -ENODEV;
 }
 
 static int _get_timeout(struct message_data *message_data)
daemons/dmeventd/libdevmapper-event.c +76 -13

103 lines not on this page · raw message

daemons/dmeventd/libdevmapper-event.h +2 -0
diff -r 1e369d480df0 -r 1c364c686b5e daemons/dmeventd/libdevmapper-event.h
@@ -102,5 +102,7 @@ void process_event(struct dm_task *dmt, 
 int register_device(const char *device_name, const char *uuid, int major, int minor, void **user);
 int unregister_device(const char *device_name, const char *uuid, int major,
 		      int minor, void **user);
+int dm_event_set_timeout(const char *dev_name, uint32_t timeout);
+int dm_event_unset_timeout(const char *dev_name);
 
 #endif
#16 lore raw

PATCH 1/1 Attempt to resync a failed secondary leg few times before giving up

In reply to Malahal Naineni

This patch adds the capability to attempt resync of a failed mirror device at a given timeout interval and given number of attempts.

Signed-off-by Malahal Naineni (malahal at us.ibm.com)
daemons/dmeventd/plugins/mirror/Makefile.in +1 -1
diff -r 1c364c686b5e -r bebd880d7476 daemons/dmeventd/plugins/mirror/Makefile.in
@@ -32,7 +32,7 @@ LIB_VERSION = $(LIB_VERSION_LVM)
 
 include $(top_builddir)/make.tmpl
 
-LIBS += -ldevmapper @LIB_PTHREAD@ @LVM2CMD_LIB@
+LIBS += -ldevmapper -ldevmapper-event @LIB_PTHREAD@ @LVM2CMD_LIB@
 
 install_lvm2: libdevmapper-event-lvm2mirror.$(LIB_SUFFIX)
 	$(INSTALL) -D $(OWNER) $(GROUP) -m 555 $(STRIP) $< \
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +84 -7

160 lines not on this page · raw message

#2 lore raw

PATCH 1/1 Add more error codes in mirror DSO

In reply to Malahal Naineni

Is there a reason you bumped ME_IGNORE from 0 to 1?

Did you forget about ME_IGNORE in 'process_event' (causing it to fall through to the final else)?

brassow

Quoted 244 lines · Malahal Naineni

The mirror DSO (daemons/dmeventd/plugins/mirror/dmeventd_mirror.c) logs
various device failure codes but only handles one failure, 'ME_FAILURE'.
This patchs adds more error codes to handle multiple types of device failures.

diff -r a48a5f0eea85 -r a74600c6163e daemons/dmeventd/plugins/mirror/ dmeventd_mirror.c
--- a/daemons/dmeventd/plugins/mirror/dmeventd_mirror.c Sun Dec 06
23:04:13 2009 -0800
+++ b/daemons/dmeventd/plugins/mirror/dmeventd_mirror.c Sun Dec 13
01:16:51 2009 -0800

(quoted diff) +0 -0
@@ -30,9 +30,17 @@
 /* FIXME Replace most syslogs with log_error() style messages and  

add complete context. */
/* FIXME Reformat to 80 char lines. */

daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +2 -3
-#define ME_IGNORE    0
-#define ME_INSYNC    1
-#define ME_FAILURE   2
+/*
+ * An event may contain more than one error type. The following are  

bit
+ * flags that indicate each error type.
+ */

daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +7 -0
+#define ME_IGNORE    			0x01U
+#define ME_INSYNC    			0x02U
+#define ME_READ_FAILURE   		0x04U
+#define ME_SYNC_FAILURE   		0x08U
+#define ME_LOG_FAILURE 			0x10U
+#define ME_SECONDARY_WRITE_FAILURE	0x20U
+#define ME_PRIMARY_WRITE_FAILURE	0x40U

/*
* register_device() is called first and performs initialisation.

daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +2 -5
@@ -53,45 +61,68 @@ static void *_lvm_handle = NULL;
 */
 static pthread_mutex_t _event_mutex = PTHREAD_MUTEX_INITIALIZER;
 
-static int _process_status_code(const char status_code, const char  
 *dev_name,
-				const char *dev_type, int r)
+static int _process_status_code(const char *mirror, const char  
 *device,
+			        const char status_code, int dev_index)
 {
-	/*
-	 *    A => Alive - No failures
-	 *    D => Dead - A write failure occurred leaving mirror out-of- 

sync

(quoted diff) +0 -2
-	 *    F => Flush failed.
-	 *    S => Sync - A sychronization failure occurred, mirror out-of- 

sync

daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +44 -17
-	 *    R => Read - A read failure occurred, mirror data unaffected
-	 *    U => Unclassified failure (bug)
-	 */
-	if (status_code == 'F') {
-		syslog(LOG_ERR, "%s device %s flush failed.\n",
-		       dev_type, dev_name);
-		r = ME_FAILURE;
-	} else if (status_code == 'S')
-		syslog(LOG_ERR, "%s device %s sync failed.\n",
-		       dev_type, dev_name);
-	else if (status_code == 'R')
-		syslog(LOG_ERR, "%s device %s read failed.\n",
-		       dev_type, dev_name);
-	else if (status_code != 'A') {
-		syslog(LOG_ERR, "%s device %s has failed (%c).\n",
-		       dev_type, dev_name, status_code);
-		r = ME_FAILURE;
+	int r;
+	const char *error_string;
+	const char *dev_type = dev_index ? "Secondary" : "Primary";
+
+	switch (status_code) {
+	default:
+	case 'U': /* Unclassified failure (bug) */
+	case 'D': /* Write failure occurred leaving mirror out-of-sync */
+	case 'F': /* Flush failure, handled as write failure */
+		if (status_code == 'D')
+			error_string = "write";
+		else if (status_code == 'F')
+			error_string = "flush";
+		else if (status_code == 'U')
+			error_string = "unclassified";
+		else {
+			syslog(LOG_ERR, "Unknown device status code (%c).\n",
+					status_code);
+			error_string = "unknown";
+		}
+
+		syslog(LOG_ERR, "Mirror device: %s, %s leg: %s had a %s "
+				"failure.\n", mirror, dev_type, device,
+				error_string);
+
+		r = dev_index? ME_SECONDARY_WRITE_FAILURE :
+			ME_PRIMARY_WRITE_FAILURE;
+		break;
+
+	case 'S': /* Sychronization failure occurred, mirror out-of-sync */
+		syslog(LOG_ERR, "Mirror device: %s, %s leg: %s had "
+				"sync failure.\n", mirror, dev_type, device);
+		r = ME_SYNC_FAILURE;
+		break;
+
+	case 'R': /* Read failure occurred, mirror data unaffected */
+		syslog(LOG_ERR, "Mirror device: %s, %s leg: %s had a "
+				"read failure.\n", mirror, dev_type, device);
+		r = ME_READ_FAILURE;
+		break;
+
+	case 'A': /* Active, a good status */
+		r = 0;
+		break;
	}

	return r;
}
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +1 -1
-static int _get_mirror_event(char *params)
+static int _get_mirror_event(const char *mirror, char *params)

{

daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +1 -1
-	int i, r = ME_INSYNC;
+	int i;
	char **args = NULL;
	char *dev_status_str;
	char *log_status_str;
	char *sync_str;
	char *p = NULL;
	int log_argc, num_devs;
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +1 -0
+	int retval = 0;
	/*
	 * dm core parms:	     0 409600 mirror
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +10 -8
@@ -122,23 +153,25 @@ static int _get_mirror_event(char *param
 
 	/* Check for bad mirror devices */
 	for (i = 0; i < num_devs; i++)
-		r = _process_status_code(dev_status_str[i], args[i],
-			i ? "Secondary mirror" : "Primary mirror", r);
+		retval |= _process_status_code(mirror, args[i],
+					       dev_status_str[i], i);
 
 	/* Check for bad disk log device */
-	if (log_argc > 1)
-		r = _process_status_code(log_status_str[0],
-					 args[2 + num_devs + log_argc],
-					 "Log", r);
+	if (log_argc > 1 && log_status_str[0] == 'D') {
+		syslog(LOG_ERR, "Mirror device: %s, log device: %s failed.\n",
+				mirror, args[2 + num_devs + log_argc]);
+		retval = retval | ME_LOG_FAILURE;
+	}
 
-	if (r == ME_FAILURE)
+	if (retval)	/* A failure occurred */
 		goto out;
 
+	retval = ME_INSYNC;	/* assume INSYNC event */
 	p = strstr(sync_str, "/");
 	if (p) {
 		p[0] = '\0';
 		if (strcmp(sync_str, p+1))
-			r = ME_IGNORE;
+			retval = ME_IGNORE;
 		p[0] = '/';
 	} else
 		goto out_parse;
@@ -146,7 +179,7 @@ static int _get_mirror_event(char *param

out:

	if (args)
		dm_free(args);
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +1 -1
-	return r;
+	return retval;

out_parse:
if (args)

daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +1 -0
@@ -212,6 +245,7 @@ void process_event(struct dm_task *dmt,
 	char *target_type = NULL;
 	char *params;
 	const char *device = dm_task_get_name(dmt);
+	int error;
 
 	if (pthread_mutex_trylock(&_event_mutex)) {
 		syslog(LOG_NOTICE, "Another thread is handling an event.   

Waiting...");

daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +5 -11
@@ -231,17 +265,11 @@ void process_event(struct dm_task *dmt,
 			continue;
 		}
 
-		switch(_get_mirror_event(params)) {
-		case ME_INSYNC:
-			/* FIXME: all we really know is that this
-			   _part_ of the device is in sync
-			   Also, this is not an error
-			*/
-			syslog(LOG_NOTICE, "%s is now in-sync\n", device);
-			break;
-		case ME_FAILURE:
-			syslog(LOG_ERR, "Device failure in %s\n", device);
-			if (_remove_failed_devices(device))
+		error = _get_mirror_event(device, params);
+		if (error & ME_LOG_FAILURE ||
+		    error & ME_PRIMARY_WRITE_FAILURE ||
+		    error & ME_SECONDARY_WRITE_FAILURE) {
+			if (_remove_failed_devices(device)) {
 				/* FIXME Why are all the error return codes unused? Get rid of  

them? */

				syslog(LOG_ERR, "Failed to remove faulty devices in %s\n",
				       device);
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +12 -8
@@ -250,13 +278,18 @@ void process_event(struct dm_task *dmt,
 				syslog(LOG_NOTICE, "%s is now a linear device.\n",
 					device);
 			*/
-			break;
-		case ME_IGNORE:
-			break;
-		default:
-			/* FIXME Provide value then! */
-			syslog(LOG_INFO, "Unknown event received.\n");
-		}
+			}
+		} else if (error & ME_INSYNC) {
+			/* FIXME: all we really know is that this
+			   _part_ of the device is in sync
+			   Also, this is not an error
+			*/
+			syslog(LOG_NOTICE, "%s is now in-sync\n", device);
+		} else if (error & ME_READ_FAILURE ||
+			   error & ME_SYNC_FAILURE) {
+			/* Ignore these for now */
+		} else
+			syslog(LOG_INFO, "Unknown event:%u received.\n", error);
 	} while (next);
 
 	pthread_mutex_unlock(&_event_mutex);
 
--

lvm-devel mailing list
lvm-devel at redhat.com
https://www.redhat.com/mailman/listinfo/lvm-devel

#3 lore raw

PATCH 1/1 Add more error codes in mirror DSO

Quoted 1 line · Jonathan Brassow [jbrassow at redhat.com]

Is there a reason you bumped ME_IGNORE from 0 to 1?

It is actually a bit-field value rather than an absolute value, so I couldn't have anything with zero.

Quoted 2 lines

Did you forget about ME_IGNORE in 'process_event' (causing it to fall through to the final else)?

Yes! Thank you. Unknown errors are handled as 'fatal' errors in the existing code, so process_event will never get an unknown error code (it has to be ME_IGNORE). I will fix it.

#6 lore raw

PATCH 1/1 Handle transient secondary mirror leg failures

In reply to Malahal Naineni

hmmm, I wonder how well this is going to work.

1) I've seen code ('dm_task_suppress_identical_reload') that suppresses table reloads if the tables are identical - will that obstruct what you are trying to do here?

2) If you don't get a new table loaded, it will behave as a suspend/ resume only. Recent code changes in dm-raid1.c are causing 'log_failure' and 'leg_failure' to not be reset in those cases. IOW, all these steps could be for nothing. :(

3) How does this work in a cluster? dmsetup is not cluster-aware. If you don't go through the LVM commands (I recognize the additional pain), none of this will be coordinated.

brassow

Quoted 19 lines · Malahal Naineni
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +7 -0
+	/* FIXME: should be running an LVM command that is pinned.
+	 * dmsetup command may not be pinned in memory all the time.
+	 * "lvchange --refresh vg/lv" only works if there are no device
+	 * failures while it is running. Otherwise, the failed device
+	 * is replaced with "error" target which is not what we want.
+	 */
+	snprintf(cmd_str, CMD_SIZE, "dmsetup suspend --noflush %s-%s", vg,  

lv);

daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +11 -0
+	syslog(LOG_NOTICE, "Running command: %s", cmd_str);
+	r = system(cmd_str);
+
+	snprintf(cmd_str, CMD_SIZE, "dmsetup table %s-%s | dmsetup load "
+			"%s-%s", vg, lv, vg, lv);
+	syslog(LOG_NOTICE, "Running command: %s", cmd_str);
+	r |= system(cmd_str);
+
+	snprintf(cmd_str, CMD_SIZE, "dmsetup resume %s-%s", vg, lv);
+	syslog(LOG_NOTICE, "Running command: %s", cmd_str);
+	r |= system(cmd_str);
#7 lore raw

PATCH 1/1 Handle transient secondary mirror leg failures

Quoted 4 lines · Jonathan Brassow

2) If you don't get a new table loaded, it will behave as a suspend/ resume only. Recent code changes in dm-raid1.c are causing 'log_failure' and 'leg_failure' to not be reset in those cases. IOW, all these steps could be for nothing. :(

I would like to know how effective the retry is. As Jon explained above, recent upstream kernel blocks all write I/Os on NOSYNC regions. This means that those write I/Os are kept blocked for a long time. For example, mirror retry interval in your patch #4 is 30 seconds and application or filesystem will be waited for 30 seconds (330 seconds if retry count is 10). Can your application wait for more than 5 minutes?

This behaviour will not been solved even if kernel is fixed so that log_failure and leg_failure are reset. The write I/Os blocked will be re-queued in the kernel when suspend/resume are done, but they will be put in the hold queue again if the device failure is not transient but permanent.

I would like to know the use case of this patch set.

Thanks,
Taka

#11 lore raw

PATCH 1/1 Handle transient secondary mirror leg failures

In reply to Jonathan Brassow
Quoted 10 lines · Jonathan Brassow [jbrassow at redhat.com]

hmmm, I wonder how well this is going to work.

1) I've seen code ('dm_task_suppress_identical_reload') that suppresses table reloads if the tables are identical - will that obstruct what you are trying to do here?

2) If you don't get a new table loaded, it will behave as a suspend/ resume only. Recent code changes in dm-raid1.c are causing 'log_failure' and 'leg_failure' to not be reset in those cases. IOW, all these steps could be for nothing. :(

Initially I did test with a modified dm-raid1.c that reset the device status (error count too) in resume() function. I wasn't using "dmsetup reload" at that time. Later, I did add "dmsetup reload" and removed my patch in dm-raid1.c.

It worked fine, so either I am not hitting dm_task_suppress_identical_reload or some how I failed to revert my patch in the kernel.

Quoted 3 lines

3) How does this work in a cluster? dmsetup is not cluster-aware. If you don't go through the LVM commands (I recognize the additional pain), none of this will be coordinated.

Tried to use "lvchange --refresh" and that didn't work too well. In the last call, Alasdair indicated I should add something like "--top-layer-only" option to "lvchange --refresh" command.

Thank you for the review.

--Malahal.

#8 lore raw

PATCH 1/1 Handle transient secondary mirror leg failures

In reply to Takahiro Yasui
Quoted 13 lines · Takahiro Yasui [tyasui at redhat.com]
Quoted 4 lines · Jonathan Brassow

2) If you don't get a new table loaded, it will behave as a suspend/ resume only. Recent code changes in dm-raid1.c are causing 'log_failure' and 'leg_failure' to not be reset in those cases. IOW, all these steps could be for nothing. :(

I would like to know how effective the retry is. As Jon explained above, recent upstream kernel blocks all write I/Os on NOSYNC regions. This means that those write I/Os are kept blocked for a long time. For example, mirror retry interval in your patch #4 is 30 seconds and application or filesystem will be waited for 30 seconds (330 seconds if retry count is 10). Can your application wait for more than 5 minutes?

Quoted 7 lines

This behaviour will not been solved even if kernel is fixed so that log_failure and leg_failure are reset. The write I/Os blocked will be re-queued in the kernel when suspend/resume are done, but they will be put in the hold queue again if the device failure is not transient but permanent.

I would like to know the use case of this patch set.

I have tested with RHEL5.4 kernel that doesn't block on device failure. IMHO, suspend/resume is doing two things if the kernel code blocks on failure -- 1) letting the kernel module unblock 2) start resync. We should separate those two actions. Maybe, we should do something here??? Have an ioctl to start resync or have a message to unblock....

Also, blocking the kernel module introduced a regression -- the mirror will block forever on a transient failure!

Thanks, Malahal.

#9 lore raw

PATCH 1/1 Handle transient secondary mirror leg failures

Quoted 20 lines · malahal at us.ibm.com
Quoted 19 lines · Takahiro Yasui [tyasui at redhat.com]
Quoted 4 lines · Jonathan Brassow

2) If you don't get a new table loaded, it will behave as a suspend/ resume only. Recent code changes in dm-raid1.c are causing 'log_failure' and 'leg_failure' to not be reset in those cases. IOW, all these steps could be for nothing. :(

I would like to know how effective the retry is. As Jon explained above, recent upstream kernel blocks all write I/Os on NOSYNC regions. This means that those write I/Os are kept blocked for a long time. For example, mirror retry interval in your patch #4 is 30 seconds and application or filesystem will be waited for 30 seconds (330 seconds if retry count is 10). Can your application wait for more than 5 minutes?

This behaviour will not been solved even if kernel is fixed so that log_failure and leg_failure are reset. The write I/Os blocked will be re-queued in the kernel when suspend/resume are done, but they will be put in the hold queue again if the device failure is not transient but permanent.

Quoted 3 lines
Quoted 1 line

I would like to know the use case of this patch set.

I have tested with RHEL5.4 kernel that doesn't block on device failure.

I see. Your retry approach looks good for the kernels on RHEL5.4 kernel and 2.6.32 kernel since write I/Os aren't blocked even after the error is detected on a mirror leg.

Quoted 4 lines

IMHO, suspend/resume is doing two things if the kernel code blocks on failure -- 1) letting the kernel module unblock 2) start resync. We should separate those two actions. Maybe, we should do something here??? Have an ioctl to start resync or have a message to unblock....

I'm sorry I don't get your point to separate unblock and resync. Currently "unblock" is done by "suspend" and resync is done by "resume" We need to reset log_failure/leg_failre, but anything else?

Quoted 2 lines

Also, blocking the kernel module introduced a regression -- the mirror will block forever on a transient failure!

You are right. Refreshing block I/Os is necessary when dmeventd noticed an error event and lvconvert command found nothing to be done for repair. dmeventd should kick an action to refresh the mirror device.

Thanks,
Taka

#10 lore raw

PATCH 1/1 Handle transient secondary mirror leg failures

Quoted 8 lines · Takahiro Yasui [tyasui at redhat.com]
Quoted 4 lines

IMHO, suspend/resume is doing two things if the kernel code blocks on failure -- 1) letting the kernel module unblock 2) start resync. We should separate those two actions. Maybe, we should do something here??? Have an ioctl to start resync or have a message to unblock....

I'm sorry I don't get your point to separate unblock and resync. Currently "unblock" is done by "suspend" and resync is done by "resume" We need to reset log_failure/leg_failre, but anything else?

Let us talk about suspend with noflush as the other case would fail the I/O. The suspend just REQUEUE's, so it really needs resume to send the I/O down to a surviving leg. As far as the application goes, the I/O gets blocked until the resume, anyway.

Also in do_writes(), we block any I/O that is going to a nosync region. If I understand, The recent block on error patch just fails to do any I/O on a "nosync" region. In other words, we don't entertain any mirror with leg failures and still continue to function as a mirror!

We can reset log_failure/leg_failure in resume to make them work for now as you say.

--Malahal.

#4 lore raw

PATCH 1/1 Add more error codes in mirror DSO

In reply to Jonathan Brassow
Quoted 6 lines · Jonathan Brassow [jbrassow at redhat.com]

Is there a reason you bumped ME_IGNORE from 0 to 1?

Did you forget about ME_IGNORE in 'process_event' (causing it to fall through to the final else)?

brassow

The following fixes the ME_IGNORE in 'process_event' [complete patch]

======================================================================

Add more error codes in mirror DSO.

The mirror DSO (daemons/dmeventd/plugins/mirror/dmeventd_mirror.c) logs various device failure codes but only handles one failure, 'ME_FAILURE'. This patchs adds more error codes to handle multiple types of device failures.

daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +90 -56
diff -r a48a5f0eea85 -r db70a48aef1b daemons/dmeventd/plugins/mirror/dmeventd_mirror.c
@@ -30,9 +30,17 @@
 /* FIXME Replace most syslogs with log_error() style messages and add complete context. */
 /* FIXME Reformat to 80 char lines. */
 
-#define ME_IGNORE    0
-#define ME_INSYNC    1
-#define ME_FAILURE   2
+/* 
+ * An event may contain more than one error type. The following are bit
+ * flags that indicate each error type.
+ */
+#define ME_IGNORE    			0x01U
+#define ME_INSYNC    			0x02U
+#define ME_READ_FAILURE   		0x04U
+#define ME_SYNC_FAILURE   		0x08U
+#define ME_LOG_FAILURE 			0x10U
+#define ME_SECONDARY_WRITE_FAILURE	0x20U
+#define ME_PRIMARY_WRITE_FAILURE	0x40U
 
 /*
  * register_device() is called first and performs initialisation.
@@ -53,45 +61,68 @@ static void *_lvm_handle = NULL;
  */
 static pthread_mutex_t _event_mutex = PTHREAD_MUTEX_INITIALIZER;
 
-static int _process_status_code(const char status_code, const char *dev_name,
-				const char *dev_type, int r)
+static int _process_status_code(const char *mirror, const char *device,
+			        const char status_code, int dev_index)
 {
-	/*
-	 *    A => Alive - No failures
-	 *    D => Dead - A write failure occurred leaving mirror out-of-sync
-	 *    F => Flush failed.
-	 *    S => Sync - A sychronization failure occurred, mirror out-of-sync
-	 *    R => Read - A read failure occurred, mirror data unaffected
-	 *    U => Unclassified failure (bug)
-	 */ 
-	if (status_code == 'F') {
-		syslog(LOG_ERR, "%s device %s flush failed.\n",
-		       dev_type, dev_name);
-		r = ME_FAILURE;
-	} else if (status_code == 'S')
-		syslog(LOG_ERR, "%s device %s sync failed.\n",
-		       dev_type, dev_name);
-	else if (status_code == 'R')
-		syslog(LOG_ERR, "%s device %s read failed.\n",
-		       dev_type, dev_name);
-	else if (status_code != 'A') {
-		syslog(LOG_ERR, "%s device %s has failed (%c).\n",
-		       dev_type, dev_name, status_code);
-		r = ME_FAILURE;
+	int r;
+	const char *error_string;
+	const char *dev_type = dev_index ? "Secondary" : "Primary";
+
+	switch (status_code) {
+	default:
+	case 'U': /* Unclassified failure (bug) */
+	case 'D': /* Write failure occurred leaving mirror out-of-sync */
+	case 'F': /* Flush failure, handled as write failure */
+		if (status_code == 'D')
+			error_string = "write";
+		else if (status_code == 'F')
+			error_string = "flush";
+		else if (status_code == 'U')
+			error_string = "unclassified";
+		else {
+			syslog(LOG_ERR, "Unknown device status code (%c).\n",
+					status_code);
+			error_string = "unknown";
+		}
+
+		syslog(LOG_ERR, "Mirror device: %s, %s leg: %s had a %s "
+				"failure.\n", mirror, dev_type, device,
+				error_string);
+
+		r = dev_index? ME_SECONDARY_WRITE_FAILURE :
+			ME_PRIMARY_WRITE_FAILURE;
+		break;
+
+	case 'S': /* Sychronization failure occurred, mirror out-of-sync */
+		syslog(LOG_ERR, "Mirror device: %s, %s leg: %s had "
+				"sync failure.\n", mirror, dev_type, device);
+		r = ME_SYNC_FAILURE;
+		break;
+
+	case 'R': /* Read failure occurred, mirror data unaffected */
+		syslog(LOG_ERR, "Mirror device: %s, %s leg: %s had a "
+				"read failure.\n", mirror, dev_type, device);
+		r = ME_READ_FAILURE;
+		break;
+
+	case 'A': /* Active, a good status */
+		r = 0;
+		break;
 	}
 
 	return r;
 }
 
-static int _get_mirror_event(char *params)
+static int _get_mirror_event(const char *mirror, char *params)
 {
-	int i, r = ME_INSYNC;
+	int i;
 	char **args = NULL;
 	char *dev_status_str;
 	char *log_status_str;
 	char *sync_str;
 	char *p = NULL;
 	int log_argc, num_devs;
+	int retval = 0;
 
 	/*
 	 * dm core parms:	     0 409600 mirror
@@ -122,23 +153,25 @@ static int _get_mirror_event(char *param
 
 	/* Check for bad mirror devices */
 	for (i = 0; i < num_devs; i++)
-		r = _process_status_code(dev_status_str[i], args[i],
-			i ? "Secondary mirror" : "Primary mirror", r);
+		retval |= _process_status_code(mirror, args[i],
+					       dev_status_str[i], i);
 
 	/* Check for bad disk log device */
-	if (log_argc > 1)
-		r = _process_status_code(log_status_str[0],
-					 args[2 + num_devs + log_argc],
-					 "Log", r);
+	if (log_argc > 1 && log_status_str[0] == 'D') {
+		syslog(LOG_ERR, "Mirror device: %s, log device: %s failed.\n",
+				mirror, args[2 + num_devs + log_argc]);
+		retval = retval | ME_LOG_FAILURE;
+	}
 
-	if (r == ME_FAILURE)
+	if (retval)	/* A failure occurred */
 		goto out;
 
+	retval = ME_INSYNC;	/* assume INSYNC event */
 	p = strstr(sync_str, "/");
 	if (p) {
 		p[0] = '\0';
 		if (strcmp(sync_str, p+1))
-			r = ME_IGNORE;
+			retval = ME_IGNORE;
 		p[0] = '/';
 	} else
 		goto out_parse;
@@ -146,7 +179,7 @@ static int _get_mirror_event(char *param
 out:
 	if (args)
 		dm_free(args);
-	return r;
+	return retval;
 	
 out_parse:
 	if (args)
@@ -212,6 +245,7 @@ void process_event(struct dm_task *dmt,
 	char *target_type = NULL;
 	char *params;
 	const char *device = dm_task_get_name(dmt);
+	int error;
 
 	if (pthread_mutex_trylock(&_event_mutex)) {
 		syslog(LOG_NOTICE, "Another thread is handling an event.  Waiting...");
@@ -231,17 +265,11 @@ void process_event(struct dm_task *dmt,
 			continue;
 		}
 
-		switch(_get_mirror_event(params)) {
-		case ME_INSYNC:
-			/* FIXME: all we really know is that this
-			   _part_ of the device is in sync
-			   Also, this is not an error
-			*/
-			syslog(LOG_NOTICE, "%s is now in-sync\n", device);
-			break;
-		case ME_FAILURE:
-			syslog(LOG_ERR, "Device failure in %s\n", device);
-			if (_remove_failed_devices(device))
+		error = _get_mirror_event(device, params);
+		if (error & ME_LOG_FAILURE ||
+		    error & ME_PRIMARY_WRITE_FAILURE ||
+		    error & ME_SECONDARY_WRITE_FAILURE) {
+			if (_remove_failed_devices(device)) {
 				/* FIXME Why are all the error return codes unused? Get rid of them? */
 				syslog(LOG_ERR, "Failed to remove faulty devices in %s\n",
 				       device);
@@ -250,13 +278,19 @@ void process_event(struct dm_task *dmt,
 				syslog(LOG_NOTICE, "%s is now a linear device.\n",
 					device);
 			*/
-			break;
-		case ME_IGNORE:
-			break;
-		default:
-			/* FIXME Provide value then! */
-			syslog(LOG_INFO, "Unknown event received.\n");
-		}
+			}
+		} else if (error & ME_INSYNC) {
+			/* FIXME: all we really know is that this
+			   _part_ of the device is in sync
+			   Also, this is not an error
+			*/
+			syslog(LOG_NOTICE, "%s is now in-sync\n", device);
+		} else if (error & ME_READ_FAILURE ||
+			   error & ME_SYNC_FAILURE ||
+			   error & ME_IGNORE) {
+			/* Ignore these for now */
+		} else
+			syslog(LOG_INFO, "Unknown event:%u received.\n", error);
 	} while (next);
 
 	pthread_mutex_unlock(&_event_mutex);
#13 lore raw

PATCH 1/1 Add dm_event_set_timeout/dm_event_unset_timeout interface

In reply to Malahal Naineni
Quoted 1 line · Malahal Naineni [malahal at us.ibm.com]

Will be used in mirror DSO to handle transient device failures.

Alasdair suggested to use -1 instead instead of INT_MAX to unregister a timeout. This patch supersedes the previous patch.

==========================================================
Add dm_event_set_timeout/dm_event_unset_timeout interface

Will be used in mirror DSO to handle transient device failures.

daemons/dmeventd/.exported_symbols +2 -0
diff -r c11c9f0a04f8 -r b3c48f241dfd daemons/dmeventd/.exported_symbols
@@ -17,3 +17,5 @@ dm_event_unregister_handler
 dm_event_get_registered_device
 dm_event_handler_set_timeout
 dm_event_handler_get_timeout
+dm_event_set_timeout
+dm_event_unset_timeout
daemons/dmeventd/dmeventd.c +19 -6
diff -r c11c9f0a04f8 -r b3c48f241dfd daemons/dmeventd/dmeventd.c
@@ -155,7 +155,7 @@ struct message_data {
 	} events;
 	union {
 		char *str;
-		uint32_t secs;
+		int32_t secs;
 	} timeout;
 	struct dm_event_daemon_message *msg;	/* Pointer to message buffer. */
 };
@@ -187,7 +187,7 @@ struct thread_status {
 	enum dm_event_mask current_events;	/* bitfield for occured events. */
 	struct dm_task *current_task;
 	time_t next_time;
-	uint32_t timeout;
+	int32_t timeout;
 	struct dm_list timeout_list;
 	void *dso_private; /* dso per-thread status variable */
 };
@@ -363,7 +363,7 @@ static int _parse_message(struct message
 			message_data->events.field = i;
 		}
 		if (message_data->timeout.str) {
-			uint32_t secs = atoi(message_data->timeout.str);
+			int32_t secs = atoi(message_data->timeout.str);
 			dm_free(message_data->timeout.str);
 			message_data->timeout.secs = secs ? secs :
 			    DM_EVENT_DEFAULT_TIMEOUT;
@@ -1164,13 +1164,26 @@ static int _get_next_registered_device(s
 static int _set_timeout(struct message_data *message_data)
 {
 	struct thread_status *thread;
+	int ret = 0;
 
 	_lock_mutex();
-	if ((thread = _lookup_thread_status(message_data)))
-		thread->timeout = message_data->timeout.secs;
+	if ((thread = _lookup_thread_status(message_data))) {
+		/* A timeout of -1 (actually any negative timeout) is
+		 * used to unregister the timeout, register a timeout
+		 * for any other value.  */
+		thread->events &= ~DM_EVENT_TIMEOUT;
+		_unregister_for_timeout(thread);
+		if (message_data->timeout.secs >= 0) {
+			thread->events |= DM_EVENT_TIMEOUT;
+			thread->timeout = message_data->timeout.secs;
+			ret = _register_for_timeout(thread);
+		} else {
+			thread->timeout = 0;
+		}
+	}
 	_unlock_mutex();
 
-	return thread ? 0 : -ENODEV;
+	return (thread && ret == 0) ? 0 : -ENODEV;
 }
 
 static int _get_timeout(struct message_data *message_data)
daemons/dmeventd/libdevmapper-event.c +79 -17

131 lines not on this page · raw message

daemons/dmeventd/libdevmapper-event.h +2 -0
diff -r c11c9f0a04f8 -r b3c48f241dfd daemons/dmeventd/libdevmapper-event.h
@@ -102,5 +102,7 @@ void process_event(struct dm_task *dmt, 
 int register_device(const char *device_name, const char *uuid, int major, int minor, void **user);
 int unregister_device(const char *device_name, const char *uuid, int major,
 		      int minor, void **user);
+int dm_event_set_timeout(const char *dev_name, int32_t timeout);
+int dm_event_unset_timeout(const char *dev_name);
 
 #endif
#14 lore raw

PATCH 1/1 Add dm_event_set_timeout/dm_event_unset_timeout interface

Quoted 2 lines · malahal at us.ibm.com

Alasdair suggested to use -1 instead instead of INT_MAX to unregister a timeout. This patch supersedes the previous patch.

Using a #define, please, rather than hard-coding -1 into the source.

Alasdair

#15 lore raw

PATCH 1/1 Add dm_event_set_timeout/dm_event_unset_timeout interface

Quoted 1 line · Alasdair G Kergon [agk at redhat.com]

Using a #define, please, rather than hard-coding -1 into the source.

Thank you. The following patch introduces DM_EVENT_REMOVE_TIMEOUT for that purpose. This patch supersedes the previous patch (both patches are against same source, so interdiff can be used to find just the difference)

=============================================================================

Add dm_event_set_timeout/dm_event_unset_timeout interface

Will be used in mirror DSO to handle transient device failures.

daemons/dmeventd/.exported_symbols +2 -0
diff -r c11c9f0a04f8 -r 0b6ad3b52702 daemons/dmeventd/.exported_symbols
@@ -17,3 +17,5 @@ dm_event_unregister_handler
 dm_event_get_registered_device
 dm_event_handler_set_timeout
 dm_event_handler_get_timeout
+dm_event_set_timeout
+dm_event_unset_timeout
daemons/dmeventd/dmeventd.c +19 -6
diff -r c11c9f0a04f8 -r 0b6ad3b52702 daemons/dmeventd/dmeventd.c
@@ -155,7 +155,7 @@ struct message_data {
 	} events;
 	union {
 		char *str;
-		uint32_t secs;
+		int32_t secs;
 	} timeout;
 	struct dm_event_daemon_message *msg;	/* Pointer to message buffer. */
 };
@@ -187,7 +187,7 @@ struct thread_status {
 	enum dm_event_mask current_events;	/* bitfield for occured events. */
 	struct dm_task *current_task;
 	time_t next_time;
-	uint32_t timeout;
+	int32_t timeout;
 	struct dm_list timeout_list;
 	void *dso_private; /* dso per-thread status variable */
 };
@@ -363,7 +363,7 @@ static int _parse_message(struct message
 			message_data->events.field = i;
 		}
 		if (message_data->timeout.str) {
-			uint32_t secs = atoi(message_data->timeout.str);
+			int32_t secs = atoi(message_data->timeout.str);
 			dm_free(message_data->timeout.str);
 			message_data->timeout.secs = secs ? secs :
 			    DM_EVENT_DEFAULT_TIMEOUT;
@@ -1164,13 +1164,26 @@ static int _get_next_registered_device(s
 static int _set_timeout(struct message_data *message_data)
 {
 	struct thread_status *thread;
+	int ret = 0;
 
 	_lock_mutex();
-	if ((thread = _lookup_thread_status(message_data)))
-		thread->timeout = message_data->timeout.secs;
+	if ((thread = _lookup_thread_status(message_data))) {
+		/* A timeout of DM_EVENT_REMOVE_TIMEOUT is used to
+		 * unregister the timeout, register a timeout for any
+		 * other value.  */
+		thread->events &= ~DM_EVENT_TIMEOUT;
+		_unregister_for_timeout(thread);
+		if (message_data->timeout.secs != DM_EVENT_REMOVE_TIMEOUT) {
+			thread->events |= DM_EVENT_TIMEOUT;
+			thread->timeout = message_data->timeout.secs;
+			ret = _register_for_timeout(thread);
+		} else {
+			thread->timeout = 0;
+		}
+	}
 	_unlock_mutex();
 
-	return thread ? 0 : -ENODEV;
+	return (thread && ret == 0) ? 0 : -ENODEV;
 }
 
 static int _get_timeout(struct message_data *message_data)
daemons/dmeventd/dmeventd.h +1 -0
diff -r c11c9f0a04f8 -r 0b6ad3b52702 daemons/dmeventd/dmeventd.h
@@ -24,6 +24,7 @@
 #define DM_EVENT_PIDFILE	"/var/run/dmeventd.pid"
 
 #define DM_EVENT_DEFAULT_TIMEOUT 10
+#define DM_EVENT_REMOVE_TIMEOUT -1	/* used to unschedule a timeout */
 
 /* Commands for the daemon passed in the message below. */
 enum dm_event_command {
daemons/dmeventd/libdevmapper-event.c +79 -17

131 lines not on this page · raw message

daemons/dmeventd/libdevmapper-event.h +2 -0
diff -r c11c9f0a04f8 -r 0b6ad3b52702 daemons/dmeventd/libdevmapper-event.h
@@ -102,5 +102,7 @@ void process_event(struct dm_task *dmt, 
 int register_device(const char *device_name, const char *uuid, int major, int minor, void **user);
 int unregister_device(const char *device_name, const char *uuid, int major,
 		      int minor, void **user);
+int dm_event_set_timeout(const char *dev_name, int32_t timeout);
+int dm_event_unset_timeout(const char *dev_name);
 
 #endif