Re-integrate a failed secondary mirror leg
PATCH 1/1 Re-integrate a failed secondary mirror leg
The following 4 patches try to re-integrate a failed secondary mirror leg.
Patch 1: Re-arranges some code and introduces more error type returns Patch 2: Tries to re-integrate the failed device indefinitely Patch 3: Removes some "#ifdef 0' code and fixes timeout setup mechanism.
Also adds interface to remove a timeout setup
Patch 4: Uses Patch 2 and Patch 3 to attempt only few times at a regular
interval to resync the failed mirror.
Tested with flakey dm module!
PATCH 1/1 Add more error codes in mirror DSO
The mirror DSO (daemons/dmeventd/plugins/mirror/dmeventd_mirror.c) logs various device failure codes but only handles one failure, 'ME_FAILURE'. This patchs adds more error codes to handle multiple types of device failures.
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +89 -56
@@ -30,9 +30,17 @@/* FIXME Replace most syslogs with log_error() style messages and add complete context. */ /* FIXME Reformat to 80 char lines. */ -#define ME_IGNORE 0 -#define ME_INSYNC 1 -#define ME_FAILURE 2 +/* +* An event may contain more than one error type. The following are bit +* flags that indicate each error type. +*/ +#define ME_IGNORE 0x01U +#define ME_INSYNC 0x02U +#define ME_READ_FAILURE 0x04U +#define ME_SYNC_FAILURE 0x08U +#define ME_LOG_FAILURE 0x10U +#define ME_SECONDARY_WRITE_FAILURE 0x20U +#define ME_PRIMARY_WRITE_FAILURE 0x40U/* * register_device() is called first and performs initialisation.
@@ -53,45 +61,68 @@ static void *_lvm_handle = NULL;*/ static pthread_mutex_t _event_mutex =PTHREAD_MUTEX_INITIALIZER ; -static int _process_status_code (const char status_code,const char *dev_name, -const char *dev_type,int r) +static int _process_status_code (const char *mirror,const char *device, +const char status_code,int dev_index) { -/* -* A => Alive - No failures -* D => Dead - A write failure occurred leaving mirror out-of-sync -* F => Flush failed. -* S => Sync - A sychronization failure occurred, mirror out-of-sync -* R => Read - A read failure occurred, mirror data unaffected -* U => Unclassified failure (bug) -*/ -if (status_code =='F' ) { -syslog (LOG_ERR ,"%s device %s flush failed.\n" , - dev_type, dev_name); - r =ME_FAILURE ; - }else if (status_code =='S' ) -syslog (LOG_ERR ,"%s device %s sync failed.\n" , - dev_type, dev_name); -else if (status_code =='R' ) -syslog (LOG_ERR ,"%s device %s read failed.\n" , - dev_type, dev_name); -else if (status_code !='A' ) { -syslog (LOG_ERR ,"%s device %s has failed (%c).\n" , - dev_type, dev_name, status_code); - r =ME_FAILURE ; +int r; +const char *error_string; +const char *dev_type = dev_index ?"Secondary" :"Primary" ; + +switch (status_code) { +default : +case 'U' :/* Unclassified failure (bug) */ +case 'D' :/* Write failure occurred leaving mirror out-of-sync */ +case 'F' :/* Flush failure, handled as write failure */ +if (status_code =='D' ) + error_string ="write" ; +else if (status_code =='F' ) + error_string ="flush" ; +else if (status_code =='U' ) + error_string ="unclassified" ; +else { +syslog (LOG_ERR ,"Unknown device status code (%c).\n" , + status_code); + error_string ="unknown" ; + } + +syslog (LOG_ERR ,"Mirror device: %s, %s leg: %s had a %s " +"failure.\n" , mirror, dev_type, device, + error_string); + + r = dev_index?ME_SECONDARY_WRITE_FAILURE : +ME_PRIMARY_WRITE_FAILURE ; +break ; + +case 'S' :/* Sychronization failure occurred, mirror out-of-sync */ +syslog (LOG_ERR ,"Mirror device: %s, %s leg: %s had " +"sync failure.\n" , mirror, dev_type, device); + r =ME_SYNC_FAILURE ; +break ; + +case 'R' :/* Read failure occurred, mirror data unaffected */ +syslog (LOG_ERR ,"Mirror device: %s, %s leg: %s had a " +"read failure.\n" , mirror, dev_type, device); + r =ME_READ_FAILURE ; +break ; + +case 'A' :/* Active, a good status */ + r =0 ; +break ; }return r; } -static int _get_mirror_event (char *params) +static int _get_mirror_event (const char *mirror,char *params) { -int i, r =ME_INSYNC ; +int i;char **args =NULL ;char *dev_status_str;char *log_status_str;char *sync_str;char *p =NULL ;int log_argc, num_devs; +int retval =0 ;/* * dm core parms: 0 409600 mirror
@@ -122,23 +153,25 @@ static int _get_mirror_event(char *param/* Check for bad mirror devices */ for (i =0 ; i < num_devs; i++) - r =_process_status_code (dev_status_str[i], args[i], - i ?"Secondary mirror" :"Primary mirror" , r); + retval |=_process_status_code (mirror, args[i], + dev_status_str[i], i);/* Check for bad disk log device */ -if (log_argc >1 ) - r =_process_status_code (log_status_str[0 ], - args[2 + num_devs + log_argc], -"Log" , r); +if (log_argc >1 && log_status_str[0 ] =='D' ) { +syslog (LOG_ERR ,"Mirror device: %s, log device: %s failed.\n" , + mirror, args[2 + num_devs + log_argc]); + retval = retval |ME_LOG_FAILURE ; + } -if (r ==ME_FAILURE ) +if (retval)/* A failure occurred */ goto out; + retval =ME_INSYNC ;/* assume INSYNC event */ p =strstr (sync_str,"/" );if (p) { p[0 ] ='\0' ;if (strcmp (sync_str, p+1 )) - r =ME_IGNORE ; + retval =ME_IGNORE ; p[0 ] ='/' ; }else goto out_parse;
@@ -146,7 +179,7 @@ static int _get_mirror_event(char *param out:if (args)dm_free (args); -return r; +return retval; out_parse:if (args)
@@ -212,6 +245,7 @@ void process_event(struct dm_task *dmt,char *target_type =NULL ;char *params;const char *device =dm_task_get_name (dmt); +int error;if (pthread_mutex_trylock (&_event_mutex)) {syslog (LOG_NOTICE ,"Another thread is handling an event. Waiting..." );
@@ -231,17 +265,11 @@ void process_event(struct dm_task *dmt,continue ; } -switch (_get_mirror_event (params)) { -case ME_INSYNC : -/* FIXME: all we really know is that this -_part_ of the device is in sync -Also, this is not an error -*/ -syslog (LOG_NOTICE ,"%s is now in-sync\n" , device); -break ; -case ME_FAILURE : -syslog (LOG_ERR ,"Device failure in %s\n" , device); -if (_remove_failed_devices (device)) + error =_get_mirror_event (device, params); +if (error &ME_LOG_FAILURE || + error &ME_PRIMARY_WRITE_FAILURE || + error &ME_SECONDARY_WRITE_FAILURE ) { +if (_remove_failed_devices (device)) {/* FIXME Why are all the error return codes unused? Get rid of them? */ syslog (LOG_ERR ,"Failed to remove faulty devices in %s\n" , device);
@@ -250,13 +278,18 @@ void process_event(struct dm_task *dmt,syslog(LOG_NOTICE, "%s is now a linear device.\n", device); */ -break ; -case ME_IGNORE : -break ; -default : -/* FIXME Provide value then! */ -syslog (LOG_INFO ,"Unknown event received.\n" ); - } + } + }else if (error &ME_INSYNC ) { +/* FIXME: all we really know is that this +_part_ of the device is in sync +Also, this is not an error +*/ +syslog (LOG_NOTICE ,"%s is now in-sync\n" , device); + }else if (error &ME_READ_FAILURE || + error &ME_SYNC_FAILURE ) { +/* Ignore these for now */ + }else +syslog (LOG_INFO ,"Unknown event:%u received.\n" , error); }while (next);pthread_mutex_unlock (&_event_mutex);
PATCH 1/1 Add more error codes in mirror DSO
Is there a reason you bumped ME_IGNORE from 0 to 1?
Did you forget about ME_IGNORE in 'process_event' (causing it to fall through to the final else)?
brassow
Quoted 244 lines · Malahal Naineni
The mirror DSO (
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c) logs
various device failure codes but only handles one failure, 'ME_FAILURE'.
This patchs adds more error codes to handle multiple types of device failures.Signed-off-by Malahal Naineni <[email protected]>diff -r
a48a5f0eea85-ra74600c6163edaemons/dmeventd/plugins/mirror/dmeventd_mirror.c
---a/daemons/dmeventd/plugins/mirror/dmeventd_mirror.cSun Dec 06
23:04:13 2009 -0800
+++b/daemons/dmeventd/plugins/mirror/dmeventd_mirror.cSun Dec 13
01:16:51 2009 -0800(quoted diff) +0
-0@@ -30,9 +30,17 @@ /* FIXME Replace most syslogs with log_error() style messages andadd complete context. */
/* FIXME Reformat to 80 char lines. */daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +2
-3-#define ME_IGNORE 0 -#define ME_INSYNC 1 -#define ME_FAILURE 2 +/* +* An event may contain more than one error type. The following are bit
+ * flags that indicate each error type.
+ */daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +7
-0+#define ME_IGNORE 0x01U +#define ME_INSYNC 0x02U +#define ME_READ_FAILURE 0x04U +#define ME_SYNC_FAILURE 0x08U +#define ME_LOG_FAILURE 0x10U +#define ME_SECONDARY_WRITE_FAILURE 0x20U +#define ME_PRIMARY_WRITE_FAILURE 0x40U/*
*register_device()is called first and performs initialisation.daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +2
-5@@ -53,45 +61,68 @@ static void *_lvm_handle = NULL;*/ static pthread_mutex_t _event_mutex =PTHREAD_MUTEX_INITIALIZER ; -static int _process_status_code (const char status_code,const char *dev_name, - const char *dev_type, int r) +static int _process_status_code(const char *mirror, const char *device, +const char status_code,int dev_index) { -/* -* A => Alive - No failures -* D => Dead - A write failure occurred leaving mirror out-of- sync
(quoted diff) +0
-2- * F => Flush failed. - * S => Sync - A sychronization failure occurred, mirror out-of-sync
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +44
-17-* R => Read - A read failure occurred, mirror data unaffected -* U => Unclassified failure (bug) -*/ -if (status_code =='F' ) { -syslog (LOG_ERR ,"%s device %s flush failed.\n" , - dev_type, dev_name); - r =ME_FAILURE ; - }else if (status_code =='S' ) -syslog (LOG_ERR ,"%s device %s sync failed.\n" , - dev_type, dev_name); -else if (status_code =='R' ) -syslog (LOG_ERR ,"%s device %s read failed.\n" , - dev_type, dev_name); -else if (status_code !='A' ) { -syslog (LOG_ERR ,"%s device %s has failed (%c).\n" , - dev_type, dev_name, status_code); - r =ME_FAILURE ; +int r; +const char *error_string; +const char *dev_type = dev_index ?"Secondary" :"Primary" ; + +switch (status_code) { +default : +case 'U' :/* Unclassified failure (bug) */ +case 'D' :/* Write failure occurred leaving mirror out-of-sync */ +case 'F' :/* Flush failure, handled as write failure */ +if (status_code =='D' ) + error_string ="write" ; +else if (status_code =='F' ) + error_string ="flush" ; +else if (status_code =='U' ) + error_string ="unclassified" ; +else { +syslog (LOG_ERR ,"Unknown device status code (%c).\n" , + status_code); + error_string ="unknown" ; + } + +syslog (LOG_ERR ,"Mirror device: %s, %s leg: %s had a %s " +"failure.\n" , mirror, dev_type, device, + error_string); + + r = dev_index?ME_SECONDARY_WRITE_FAILURE : +ME_PRIMARY_WRITE_FAILURE ; +break ; + +case 'S' :/* Sychronization failure occurred, mirror out-of-sync */ +syslog (LOG_ERR ,"Mirror device: %s, %s leg: %s had " +"sync failure.\n" , mirror, dev_type, device); + r =ME_SYNC_FAILURE ; +break ; + +case 'R' :/* Read failure occurred, mirror data unaffected */ +syslog (LOG_ERR ,"Mirror device: %s, %s leg: %s had a " +"read failure.\n" , mirror, dev_type, device); + r =ME_READ_FAILURE ; +break ; + +case 'A' :/* Active, a good status */ + r =0 ; +break ;} return r; }daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +1
-1-static int _get_mirror_event (char *params) +static int _get_mirror_event (const char *mirror,char *params){
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +1
-1-int i, r =ME_INSYNC ; +int i;char **args = NULL; char *dev_status_str; char *log_status_str; char *sync_str; char *p = NULL; int log_argc, num_devs;daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +1
-0+int retval =0 ;/* * dm core parms: 0 409600 mirrordaemons/dmeventd/plugins/mirror/dmeventd_mirror.c +10
-8@@ -122,23 +153,25 @@ static int _get_mirror_event(char *param/* Check for bad mirror devices */ for (i =0 ; i < num_devs; i++) - r =_process_status_code (dev_status_str[i], args[i], - i ?"Secondary mirror" :"Primary mirror" , r); + retval |=_process_status_code (mirror, args[i], + dev_status_str[i], i);/* Check for bad disk log device */ -if (log_argc >1 ) - r =_process_status_code (log_status_str[0 ], - args[2 + num_devs + log_argc], -"Log" , r); +if (log_argc >1 && log_status_str[0 ] =='D' ) { +syslog (LOG_ERR ,"Mirror device: %s, log device: %s failed.\n" , + mirror, args[2 + num_devs + log_argc]); + retval = retval |ME_LOG_FAILURE ; + } -if (r ==ME_FAILURE ) +if (retval)/* A failure occurred */ goto out; + retval =ME_INSYNC ;/* assume INSYNC event */ p =strstr (sync_str,"/" );if (p) { p[0 ] ='\0' ;if (strcmp (sync_str, p+1 )) - r =ME_IGNORE ; + retval =ME_IGNORE ; p[0 ] ='/' ; }else goto out_parse;@@ -146,7 +179,7 @@ static int _get_mirror_event(char *paramout:
if (args) dm_free(args);daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +1
-1-return r; +return retval;
out_parse:
if (args)daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +1
-0@@ -212,6 +245,7 @@ void process_event(struct dm_task *dmt,char *target_type =NULL ;char *params;const char *device =dm_task_get_name (dmt); +int error;if (pthread_mutex_trylock (&_event_mutex)) { syslog(LOG_NOTICE, "Another thread is handling an event.Waiting...");
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +5
-11@@ -231,17 +265,11 @@ void process_event(struct dm_task *dmt,continue ; } -switch (_get_mirror_event (params)) { -case ME_INSYNC : -/* FIXME: all we really know is that this -_part_ of the device is in sync -Also, this is not an error -*/ -syslog (LOG_NOTICE ,"%s is now in-sync\n" , device); -break ; -case ME_FAILURE : -syslog (LOG_ERR ,"Device failure in %s\n" , device); -if (_remove_failed_devices (device)) + error =_get_mirror_event (device, params); +if (error &ME_LOG_FAILURE || + error &ME_PRIMARY_WRITE_FAILURE || + error &ME_SECONDARY_WRITE_FAILURE ) { +if (_remove_failed_devices (device)) {/* FIXME Why are all the error return codes unused? Get rid of them? */
syslog(LOG_ERR, "Failed to remove faulty devices in %s\n", device);daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +12
-8@@ -250,13 +278,18 @@ void process_event(struct dm_task *dmt,syslog(LOG_NOTICE, "%s is now a linear device.\n", device); */ -break ; -case ME_IGNORE : -break ; -default : -/* FIXME Provide value then! */ -syslog (LOG_INFO ,"Unknown event received.\n" ); - } + } + }else if (error &ME_INSYNC ) { +/* FIXME: all we really know is that this +_part_ of the device is in sync +Also, this is not an error +*/ +syslog (LOG_NOTICE ,"%s is now in-sync\n" , device); + }else if (error &ME_READ_FAILURE || + error &ME_SYNC_FAILURE ) { +/* Ignore these for now */ + }else +syslog (LOG_INFO ,"Unknown event:%u received.\n" , error); }while (next);pthread_mutex_unlock (&_event_mutex); --lvm-devel mailing list
lvm-devel at redhat.com
https://www.redhat.com/mailman/listinfo/lvm-devel
PATCH 1/1 Add more error codes in mirror DSO
Quoted 1 line · Jonathan Brassow [jbrassow at redhat.com]
Is there a reason you bumped
ME_IGNOREfrom 0 to 1?
It is actually a bit-field value rather than an absolute value, so I couldn't have anything with zero.
Quoted 2 lines
Did you forget about
ME_IGNOREin 'process_event' (causing it to fall through to the final else)?
Yes! Thank you. Unknown errors are handled as 'fatal' errors in the existing code, so process_event will never get an unknown error code (it has to be ME_IGNORE). I will fix it.
PATCH 1/1 Add more error codes in mirror DSO
In reply to Jonathan BrassowQuoted 6 lines · Jonathan Brassow [jbrassow at redhat.com]
Is there a reason you bumped
ME_IGNOREfrom 0 to 1?Did you forget about
ME_IGNOREin 'process_event' (causing it to fall through to the final else)?brassow
The following fixes the ME_IGNORE in 'process_event' [complete patch]
======================================================================
Add more error codes in mirror DSO.
The mirror DSO (daemons/dmeventd/plugins/mirror/dmeventd_mirror.c) logs various device failure codes but only handles one failure, 'ME_FAILURE'. This patchs adds more error codes to handle multiple types of device failures.
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +90 -56
@@ -30,9 +30,17 @@/* FIXME Replace most syslogs with log_error() style messages and add complete context. */ /* FIXME Reformat to 80 char lines. */ -#define ME_IGNORE 0 -#define ME_INSYNC 1 -#define ME_FAILURE 2 +/* +* An event may contain more than one error type. The following are bit +* flags that indicate each error type. +*/ +#define ME_IGNORE 0x01U +#define ME_INSYNC 0x02U +#define ME_READ_FAILURE 0x04U +#define ME_SYNC_FAILURE 0x08U +#define ME_LOG_FAILURE 0x10U +#define ME_SECONDARY_WRITE_FAILURE 0x20U +#define ME_PRIMARY_WRITE_FAILURE 0x40U/* * register_device() is called first and performs initialisation.
@@ -53,45 +61,68 @@ static void *_lvm_handle = NULL;*/ static pthread_mutex_t _event_mutex =PTHREAD_MUTEX_INITIALIZER ; -static int _process_status_code (const char status_code,const char *dev_name, -const char *dev_type,int r) +static int _process_status_code (const char *mirror,const char *device, +const char status_code,int dev_index) { -/* -* A => Alive - No failures -* D => Dead - A write failure occurred leaving mirror out-of-sync -* F => Flush failed. -* S => Sync - A sychronization failure occurred, mirror out-of-sync -* R => Read - A read failure occurred, mirror data unaffected -* U => Unclassified failure (bug) -*/ -if (status_code =='F' ) { -syslog (LOG_ERR ,"%s device %s flush failed.\n" , - dev_type, dev_name); - r =ME_FAILURE ; - }else if (status_code =='S' ) -syslog (LOG_ERR ,"%s device %s sync failed.\n" , - dev_type, dev_name); -else if (status_code =='R' ) -syslog (LOG_ERR ,"%s device %s read failed.\n" , - dev_type, dev_name); -else if (status_code !='A' ) { -syslog (LOG_ERR ,"%s device %s has failed (%c).\n" , - dev_type, dev_name, status_code); - r =ME_FAILURE ; +int r; +const char *error_string; +const char *dev_type = dev_index ?"Secondary" :"Primary" ; + +switch (status_code) { +default : +case 'U' :/* Unclassified failure (bug) */ +case 'D' :/* Write failure occurred leaving mirror out-of-sync */ +case 'F' :/* Flush failure, handled as write failure */ +if (status_code =='D' ) + error_string ="write" ; +else if (status_code =='F' ) + error_string ="flush" ; +else if (status_code =='U' ) + error_string ="unclassified" ; +else { +syslog (LOG_ERR ,"Unknown device status code (%c).\n" , + status_code); + error_string ="unknown" ; + } + +syslog (LOG_ERR ,"Mirror device: %s, %s leg: %s had a %s " +"failure.\n" , mirror, dev_type, device, + error_string); + + r = dev_index?ME_SECONDARY_WRITE_FAILURE : +ME_PRIMARY_WRITE_FAILURE ; +break ; + +case 'S' :/* Sychronization failure occurred, mirror out-of-sync */ +syslog (LOG_ERR ,"Mirror device: %s, %s leg: %s had " +"sync failure.\n" , mirror, dev_type, device); + r =ME_SYNC_FAILURE ; +break ; + +case 'R' :/* Read failure occurred, mirror data unaffected */ +syslog (LOG_ERR ,"Mirror device: %s, %s leg: %s had a " +"read failure.\n" , mirror, dev_type, device); + r =ME_READ_FAILURE ; +break ; + +case 'A' :/* Active, a good status */ + r =0 ; +break ; }return r; } -static int _get_mirror_event (char *params) +static int _get_mirror_event (const char *mirror,char *params) { -int i, r =ME_INSYNC ; +int i;char **args =NULL ;char *dev_status_str;char *log_status_str;char *sync_str;char *p =NULL ;int log_argc, num_devs; +int retval =0 ;/* * dm core parms: 0 409600 mirror
@@ -122,23 +153,25 @@ static int _get_mirror_event(char *param/* Check for bad mirror devices */ for (i =0 ; i < num_devs; i++) - r =_process_status_code (dev_status_str[i], args[i], - i ?"Secondary mirror" :"Primary mirror" , r); + retval |=_process_status_code (mirror, args[i], + dev_status_str[i], i);/* Check for bad disk log device */ -if (log_argc >1 ) - r =_process_status_code (log_status_str[0 ], - args[2 + num_devs + log_argc], -"Log" , r); +if (log_argc >1 && log_status_str[0 ] =='D' ) { +syslog (LOG_ERR ,"Mirror device: %s, log device: %s failed.\n" , + mirror, args[2 + num_devs + log_argc]); + retval = retval |ME_LOG_FAILURE ; + } -if (r ==ME_FAILURE ) +if (retval)/* A failure occurred */ goto out; + retval =ME_INSYNC ;/* assume INSYNC event */ p =strstr (sync_str,"/" );if (p) { p[0 ] ='\0' ;if (strcmp (sync_str, p+1 )) - r =ME_IGNORE ; + retval =ME_IGNORE ; p[0 ] ='/' ; }else goto out_parse;
@@ -146,7 +179,7 @@ static int _get_mirror_event(char *param out:if (args)dm_free (args); -return r; +return retval; out_parse:if (args)
@@ -212,6 +245,7 @@ void process_event(struct dm_task *dmt,char *target_type =NULL ;char *params;const char *device =dm_task_get_name (dmt); +int error;if (pthread_mutex_trylock (&_event_mutex)) {syslog (LOG_NOTICE ,"Another thread is handling an event. Waiting..." );
@@ -231,17 +265,11 @@ void process_event(struct dm_task *dmt,continue ; } -switch (_get_mirror_event (params)) { -case ME_INSYNC : -/* FIXME: all we really know is that this -_part_ of the device is in sync -Also, this is not an error -*/ -syslog (LOG_NOTICE ,"%s is now in-sync\n" , device); -break ; -case ME_FAILURE : -syslog (LOG_ERR ,"Device failure in %s\n" , device); -if (_remove_failed_devices (device)) + error =_get_mirror_event (device, params); +if (error &ME_LOG_FAILURE || + error &ME_PRIMARY_WRITE_FAILURE || + error &ME_SECONDARY_WRITE_FAILURE ) { +if (_remove_failed_devices (device)) {/* FIXME Why are all the error return codes unused? Get rid of them? */ syslog (LOG_ERR ,"Failed to remove faulty devices in %s\n" , device);
@@ -250,13 +278,19 @@ void process_event(struct dm_task *dmt,syslog(LOG_NOTICE, "%s is now a linear device.\n", device); */ -break ; -case ME_IGNORE : -break ; -default : -/* FIXME Provide value then! */ -syslog (LOG_INFO ,"Unknown event received.\n" ); - } + } + }else if (error &ME_INSYNC ) { +/* FIXME: all we really know is that this +_part_ of the device is in sync +Also, this is not an error +*/ +syslog (LOG_NOTICE ,"%s is now in-sync\n" , device); + }else if (error &ME_READ_FAILURE || + error &ME_SYNC_FAILURE || + error &ME_IGNORE ) { +/* Ignore these for now */ + }else +syslog (LOG_INFO ,"Unknown event:%u received.\n" , error); }while (next);pthread_mutex_unlock (&_event_mutex);
PATCH 1/1 Handle transient secondary mirror leg failures
In reply to Malahal NaineniA new mirror_device_fault_policy, "retry", is added to handle transient device failures. When this policy is selected, the mirror DSO will try to resync the mirror upon a secondary leg failure. This will be tried and tried until the mirror goes to in_sync state. Later patches make this a configurable number spaced at some configurable timeout.
The patch uses dmsetup suspend and resume commands to attempt a resync. It uses "lvm dumpconfig" to find out the mirror_device_fault_policy.
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +179 -9
@@ -14,6 +14,7 @@#include "lvm2cmd.h" #include "errors.h" +#include "defaults.h" #include <libdevmapper.h> #include <libdevmapper-event.h>
@@ -24,6 +25,7 @@#include <stdlib.h> #include <pthread.h> #include <unistd.h> +#include <ctype.h> #include <syslog.h> /* FIXME Replace syslog with multilog */ /* FIXME Missing openlog? */
@@ -56,6 +58,109 @@ static int _register_count = 0;static struct dm_pool *_mem_pool =NULL ;static void *_lvm_handle =NULL ; +enum fault_policy { +FAULT_POLICY_INVALID , +FAULT_POLICY_REMOVE , +FAULT_POLICY_ALLOCATE , +FAULT_POLICY_ALLOCATE_ANYWHERE , +FAULT_POLICY_RETRY , +}; + +struct mirror_device_info { +enum fault_policy fault_policy; +}; + +#define CMD_SIZE 256/* FIXME Use system restriction */ +#define LINE_SIZE 1024/* FIXME Use system restriction */ +static int fill_config_str (char buf[],int bufsize,const char *path) +{ +char cmd_str[CMD_SIZE ]; +char data[LINE_SIZE ], *ptr; +int ret =0 ; +FILE *fp; + +snprintf (cmd_str,sizeof (cmd_str),"lvm dumpconfig %s" , path); +if ((fp =popen (cmd_str,"r" )) ==NULL ) { +syslog (LOG_ERR ,"fopen() failed.\n" ); + goto out; + } + +if ((ptr =fgets (data,sizeof (data), fp)) ==NULL ) { +syslog (LOG_ERR ,"fgets() failed.\n" ); +pclose (fp); + goto out; + } + +/* Remove white space or quotes at the end */ +for (ptr = &data[strlen (data)-1 ]; ptr >= data; --ptr) +if (isspace (*ptr) || *ptr =='\"' || *ptr =='\'' ) + *ptr ='\0' ; +else +break ; + ptr =strchr (data,'=' ); +if (ptr) { + ptr++;/* After the '=' sign */ +/* skip quotes or white space */ +if (isspace (*ptr) || *ptr =='\"' || *ptr =='\'' ) + ptr++; +strncpy (buf, ptr, bufsize); + buf[bufsize-1 ] ='\0' ; + ret =1 ;/* buf is valid */ + } +if (pclose (fp) !=0 ) +syslog (LOG_ERR ,"pclose() failed.\n" ); + +out: +return ret; +} + +static enum fault_policy fault_policy_str2enum (const char *str) +{ +enum fault_policy ret =FAULT_POLICY_INVALID ; + +/* At most the loop is ran twice */ +while (ret ==FAULT_POLICY_INVALID ) { +if (!strcmp (str,"remove" )) + ret =FAULT_POLICY_REMOVE ; +else if (!strcmp (str,"allocate" )) + ret =FAULT_POLICY_ALLOCATE ; +else if (!strcmp (str,"allocate_anywhere" )) + ret =FAULT_POLICY_ALLOCATE_ANYWHERE ; +else if (!strcmp (str,"retry" )) + ret =FAULT_POLICY_RETRY ; +else { +syslog (LOG_ERR ,"Bad activation/" +"mirror_device_fault_policy: %s\n" , + str); + str =DEFAULT_MIRROR_DEV_FAULT_POLICY ; + } + } + +return ret; +} + + +static enum fault_policy get_mirror_fault_policy () +{ +enum fault_policy ret; +char policy[LINE_SIZE ]; +const char *ptr; + + ret =fill_config_str (policy,sizeof (policy), +"activation/mirror_device_fault_policy" ); +if (ret) { +if (!strcmp (policy,"" )) + ptr =DEFAULT_MIRROR_DEV_FAULT_POLICY ; +else + ptr = policy; + }else { + ptr =DEFAULT_MIRROR_DEV_FAULT_POLICY ; + } + ret =fault_policy_str2enum (ptr); + +return ret; +} +/* * Currently only one event can be processed at a time. */
@@ -200,10 +305,52 @@ static void _temporary_log_fn(int level,syslog (LOG_DEBUG ,"%s" , format); } + +static int retry_failed_devices (const char *device) +{ +int r; +char cmd_str[CMD_SIZE ]; +char *vg =NULL , *lv =NULL , *layer =NULL ; + +if (strlen (device) >200 )/* FIXME Use real restriction */ +/* FIXME These return code distinctions are not used so +* remove them! */ +return -ENAMETOOLONG ; + +if (!dm_split_lvm_name (_mem_pool, device, &vg, &lv, &layer)) { +syslog (LOG_ERR ,"Unable to determine VG name from %s" , + device); +/* FIXME Replace with generic error return - reason for +* failure has already got logged */ +return -ENOMEM ; + } + +/* FIXME: should be running an LVM command that is pinned. +* dmsetup command may not be pinned in memory all the time. +* "lvchange --refresh vg/lv" only works if there are no device +* failures while it is running. Otherwise, the failed device +* is replaced with "error" target which is not what we want. +*/ +snprintf (cmd_str,CMD_SIZE ,"dmsetup suspend --noflush %s-%s" , vg, lv); +syslog (LOG_NOTICE ,"Running command: %s" , cmd_str); + r =system (cmd_str); + +snprintf (cmd_str,CMD_SIZE ,"dmsetup table %s-%s | dmsetup load " +"%s-%s" , vg, lv, vg, lv); +syslog (LOG_NOTICE ,"Running command: %s" , cmd_str); + r |=system (cmd_str); + +snprintf (cmd_str,CMD_SIZE ,"dmsetup resume %s-%s" , vg, lv); +syslog (LOG_NOTICE ,"Running command: %s" , cmd_str); + r |=system (cmd_str); + +dm_pool_empty (_mem_pool);/* FIXME: not safe with multiple threads */ +return r; +} +static int _remove_failed_devices (const char *device) {int r; -#define CMD_SIZE 256/* FIXME Use system restriction */ char cmd_str[CMD_SIZE ];char *vg =NULL , *lv =NULL , *layer =NULL ;
@@ -238,7 +385,7 @@ static int _remove_failed_devices(constvoid process_event (struct dm_task *dmt,enum dm_event_mask event __attribute((unused)), -void **unused __attribute((unused))) +void **private) {void *next =NULL ;uint64_t start, length;
@@ -246,6 +393,7 @@ void process_event(struct dm_task *dmt,char *params;const char *device =dm_task_get_name (dmt);int error; +struct mirror_device_info *mirror_info = *private;if (pthread_mutex_trylock (&_event_mutex)) {syslog (LOG_NOTICE ,"Another thread is handling an event. Waiting..." );
@@ -268,8 +416,17 @@ void process_event(struct dm_task *dmt, error =_get_mirror_event (device, params);if (error &ME_LOG_FAILURE || error &ME_PRIMARY_WRITE_FAILURE || - error &ME_SECONDARY_WRITE_FAILURE ) { -if (_remove_failed_devices (device)) { + error &ME_SECONDARY_WRITE_FAILURE || + error &ME_SYNC_FAILURE ) { +if (mirror_info->fault_policy ==FAULT_POLICY_RETRY && + (error &ME_SECONDARY_WRITE_FAILURE || + error &ME_SYNC_FAILURE )) { +syslog (LOG_ERR ,"Retrying the failed mirror " +"device.\n" ); +if (retry_failed_devices (device)) +syslog (LOG_ERR ,"Failed to reload the " +"mirror: %s\n" , device); + }else if (_remove_failed_devices (device)) {/* FIXME Why are all the error return codes unused? Get rid of them? */ syslog (LOG_ERR ,"Failed to remove faulty devices in %s\n" , device);
@@ -285,9 +442,8 @@ void process_event(struct dm_task *dmt,Also, this is not an error */ syslog (LOG_NOTICE ,"%s is now in-sync\n" , device); - }else if (error &ME_READ_FAILURE || - error &ME_SYNC_FAILURE ) { -/* Ignore these for now */ + }else if (error &ME_READ_FAILURE ) { +/* Ignore it for now */ }else syslog (LOG_INFO ,"Unknown event:%u received.\n" , error); }while (next);
@@ -299,9 +455,10 @@ int register_device(const char *device, const char *uuid __attribute((unused)), int major __attribute((unused)), int minor __attribute((unused)), - void **unused __attribute((unused))) + void **private) {int r =0 ; +struct mirror_device_info *mirror_info;pthread_mutex_lock (&_register_mutex);
@@ -312,9 +469,19 @@ int register_device(const char *device,if (!_mem_pool && !(_mem_pool =dm_pool_create ("mirror_dso" ,1024 ))) goto out; + mirror_info =dm_malloc (sizeof (struct mirror_device_info )); +if (!mirror_info) { +dm_pool_destroy (_mem_pool); + _mem_pool =NULL ; + goto out; + } + mirror_info->fault_policy =get_mirror_fault_policy (); + *private = mirror_info; +if (!_lvm_handle) {lvm2_log_fn (_temporary_log_fn);if (!(_lvm_handle =lvm2_init ())) { +dm_free (mirror_info);dm_pool_destroy (_mem_pool); _mem_pool =NULL ; goto out;
@@ -339,8 +506,11 @@ int unregister_device(const char *device const char *uuid __attribute((unused)), int major __attribute((unused)), int minor __attribute((unused)), - void **unused __attribute((unused))) + void **private) { +struct mirror_device_info *mirror_info = *private; + +dm_free (mirror_info);pthread_mutex_lock (&_register_mutex); syslog(LOG_INFO, "No longer monitoring mirror device %s for events\n",
doc/example.conf +3 -0
@@ -403,6 +403,9 @@ activation { # since it would break the redundant nature of the mirror. This # policy acts like "remove" if no suitable device and space can # be allocated for the replacement. + # + # "retry" - Try to re-integrate the failed mirror leg assuming that the + # failure is transient. Not implemented yet, so don't use it. mirror_log_fault_policy = "allocate" mirror_device_fault_policy = "remove"
lib/metadata/mirror.c +3 -0
@@ -37,6 +37,7 @@#define MIRROR_REMOVE 0#define MIRROR_ALLOCATE 1#define MIRROR_ALLOCATE_ANYWHERE 2 +#define MIRROR_RETRY 3/* * Returns true if the lv is temporary mirror layer for resync
@@ -787,6 +788,8 @@ static int get_mirror_fault_policy(strucreturn MIRROR_ALLOCATE ; else if (!strcmp(policy, "allocate_anywhere"))return MIRROR_ALLOCATE_ANYWHERE ; + else if (!strcmp(policy, "retry")) +return MIRROR_RETRY ;if (log_policy)log_error ("Bad activation/mirror_log_fault_policy" );
PATCH 1/1 Handle transient secondary mirror leg failures
hmmm, I wonder how well this is going to work.
1) I've seen code ('dm_task_suppress_identical_reload') that suppresses table reloads if the tables are identical - will that obstruct what you are trying to do here?
2) If you don't get a new table loaded, it will behave as a suspend/ resume only. Recent code changes in dm-raid1.c are causing 'log_failure' and 'leg_failure' to not be reset in those cases. IOW, all these steps could be for nothing. :(
3) How does this work in a cluster? dmsetup is not cluster-aware. If you don't go through the LVM commands (I recognize the additional pain), none of this will be coordinated.
brassow
Quoted 19 lines · Malahal Naineni
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +7
-0+/* FIXME: should be running an LVM command that is pinned. +* dmsetup command may not be pinned in memory all the time. +* "lvchange --refresh vg/lv" only works if there are no device +* failures while it is running. Otherwise, the failed device +* is replaced with "error" target which is not what we want. +*/ + snprintf(cmd_str, CMD_SIZE, "dmsetup suspend --noflush %s-%s", vg,lv);
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +11
-0+syslog (LOG_NOTICE ,"Running command: %s" , cmd_str); + r =system (cmd_str); + +snprintf (cmd_str,CMD_SIZE ,"dmsetup table %s-%s | dmsetup load " +"%s-%s" , vg, lv, vg, lv); +syslog (LOG_NOTICE ,"Running command: %s" , cmd_str); + r |=system (cmd_str); + +snprintf (cmd_str,CMD_SIZE ,"dmsetup resume %s-%s" , vg, lv); +syslog (LOG_NOTICE ,"Running command: %s" , cmd_str); + r |=system (cmd_str);
PATCH 1/1 Handle transient secondary mirror leg failures
Quoted 4 lines · Jonathan Brassow
2) If you don't get a new table loaded, it will behave as a suspend/ resume only. Recent code changes in
dm-raid1.care causing 'log_failure' and 'leg_failure' to not be reset in those cases. IOW, all these steps could be for nothing. :(
I would like to know how effective the retry is. As Jon explained above, recent upstream kernel blocks all write I/Os on NOSYNC regions. This means that those write I/Os are kept blocked for a long time. For example, mirror retry interval in your patch #4 is 30 seconds and application or filesystem will be waited for 30 seconds (330 seconds if retry count is 10). Can your application wait for more than 5 minutes?
This behaviour will not been solved even if kernel is fixed so that log_failure and leg_failure are reset. The write I/Os blocked will be re-queued in the kernel when suspend/resume are done, but they will be put in the hold queue again if the device failure is not transient but permanent.
I would like to know the use case of this patch set.
Thanks,
Taka
PATCH 1/1 Handle transient secondary mirror leg failures
Quoted 13 lines · Takahiro Yasui [tyasui at redhat.com]
Quoted 4 lines · Jonathan Brassow
2) If you don't get a new table loaded, it will behave as a suspend/ resume only. Recent code changes in
dm-raid1.care causing 'log_failure' and 'leg_failure' to not be reset in those cases. IOW, all these steps could be for nothing. :(I would like to know how effective the retry is. As Jon explained above, recent upstream kernel blocks all write I/Os on NOSYNC regions. This means that those write I/Os are kept blocked for a long time. For example, mirror retry interval in your patch #4 is 30 seconds and application or filesystem will be waited for 30 seconds (330 seconds if retry count is 10). Can your application wait for more than 5 minutes?
Quoted 7 lines
This behaviour will not been solved even if kernel is fixed so that
log_failureandleg_failureare reset. The write I/Os blocked will be re-queued in the kernel when suspend/resume are done, but they will be put in the hold queue again if the device failure is not transient but permanent.I would like to know the use case of this patch set.
I have tested with RHEL5.4 kernel that doesn't block on device failure. IMHO, suspend/resume is doing two things if the kernel code blocks on failure -- 1) letting the kernel module unblock 2) start resync. We should separate those two actions. Maybe, we should do something here??? Have an ioctl to start resync or have a message to unblock....
Also, blocking the kernel module introduced a regression -- the mirror will block forever on a transient failure!
Thanks, Malahal.
PATCH 1/1 Handle transient secondary mirror leg failures
Quoted 20 lines · malahal at us.ibm.com
Quoted 19 lines · Takahiro Yasui [tyasui at redhat.com]
Quoted 4 lines · Jonathan Brassow
2) If you don't get a new table loaded, it will behave as a suspend/ resume only. Recent code changes in
dm-raid1.care causing 'log_failure' and 'leg_failure' to not be reset in those cases. IOW, all these steps could be for nothing. :(I would like to know how effective the retry is. As Jon explained above, recent upstream kernel blocks all write I/Os on NOSYNC regions. This means that those write I/Os are kept blocked for a long time. For example, mirror retry interval in your patch #4 is 30 seconds and application or filesystem will be waited for 30 seconds (330 seconds if retry count is 10). Can your application wait for more than 5 minutes?
This behaviour will not been solved even if kernel is fixed so that
log_failureandleg_failureare reset. The write I/Os blocked will be re-queued in the kernel when suspend/resume are done, but they will be put in the hold queue again if the device failure is not transient but permanent.
Quoted 3 lines
Quoted 1 line
I would like to know the use case of this patch set.
I have tested with RHEL5.4 kernel that doesn't block on device failure.
I see. Your retry approach looks good for the kernels on RHEL5.4 kernel and 2.6.32 kernel since write I/Os aren't blocked even after the error is detected on a mirror leg.
Quoted 4 lines
IMHO, suspend/resume is doing two things if the kernel code blocks on failure -- 1) letting the kernel module unblock 2) start resync. We should separate those two actions. Maybe, we should do something here??? Have an ioctl to start resync or have a message to unblock....
I'm sorry I don't get your point to separate unblock and resync. Currently "unblock" is done by "suspend" and resync is done by "resume" We need to reset log_failure/leg_failre, but anything else?
Quoted 2 lines
Also, blocking the kernel module introduced a regression -- the mirror will block forever on a transient failure!
You are right. Refreshing block I/Os is necessary when dmeventd noticed an error event and lvconvert command found nothing to be done for repair. dmeventd should kick an action to refresh the mirror device.
Thanks,
Taka
PATCH 1/1 Handle transient secondary mirror leg failures
Quoted 8 lines · Takahiro Yasui [tyasui at redhat.com]
Quoted 4 lines
IMHO, suspend/resume is doing two things if the kernel code blocks on failure -- 1) letting the kernel module unblock 2) start resync. We should separate those two actions. Maybe, we should do something here??? Have an ioctl to start resync or have a message to unblock....
I'm sorry I don't get your point to separate unblock and resync. Currently "unblock" is done by "suspend" and resync is done by "resume" We need to reset log_failure/leg_failre, but anything else?
Let us talk about suspend with noflush as the other case would fail the I/O. The suspend just REQUEUE's, so it really needs resume to send the I/O down to a surviving leg. As far as the application goes, the I/O gets blocked until the resume, anyway.
Also in do_writes(), we block any I/O that is going to a nosync region. If I understand, The recent block on error patch just fails to do any I/O on a "nosync" region. In other words, we don't entertain any mirror with leg failures and still continue to function as a mirror!
We can reset log_failure/leg_failure in resume to make them work for now as you say.
--Malahal.
PATCH 1/1 Handle transient secondary mirror leg failures
In reply to Jonathan BrassowQuoted 10 lines · Jonathan Brassow [jbrassow at redhat.com]
hmmm, I wonder how well this is going to work.
1) I've seen code ('
dm_task_suppress_identical_reload') that suppresses table reloads if the tables are identical - will that obstruct what you are trying to do here?2) If you don't get a new table loaded, it will behave as a suspend/ resume only. Recent code changes in
dm-raid1.care causing 'log_failure' and 'leg_failure' to not be reset in those cases. IOW, all these steps could be for nothing. :(
Initially I did test with a modified dm-raid1.c that reset the device status (error count too) in resume() function. I wasn't using "dmsetup reload" at that time. Later, I did add "dmsetup reload" and removed my patch in dm-raid1.c.
It worked fine, so either I am not hitting dm_task_suppress_identical_reload or some how I failed to revert my patch in the kernel.
Quoted 3 lines
3) How does this work in a cluster? dmsetup is not cluster-aware. If you don't go through the LVM commands (I recognize the additional pain), none of this will be coordinated.
Tried to use "lvchange --refresh" and that didn't work too well. In the last call, Alasdair indicated I should add something like "--top-layer-only" option to "lvchange --refresh" command.
Thank you for the review.
--Malahal.
PATCH 1/1 Add dm_event_set_timeout/dm_event_unset_timeout interface
In reply to Malahal NaineniWill be used in mirror DSO to handle transient device failures.
daemons/dmeventd/.exported_symbols +2 -0
@@ -17,3 +17,5 @@ dm_event_unregister_handler dm_event_get_registered_device dm_event_handler_set_timeout dm_event_handler_get_timeout +dm_event_set_timeout +dm_event_unset_timeout
daemons/dmeventd/dmeventd.c +15 -3
@@ -1164,13 +1164,25 @@ static int _get_next_registered_device(sstatic int _set_timeout (struct message_data *message_data) {struct thread_status *thread; +int ret =0 ;_lock_mutex (); -if ((thread =_lookup_thread_status (message_data))) - thread->timeout = message_data->timeout.secs; +if ((thread =_lookup_thread_status (message_data))) { +/* INT_MAX timeout is used to unregister timeout, +* register timeout for any other value. */ + thread->events &= ~DM_EVENT_TIMEOUT ; +_unregister_for_timeout (thread); +if (message_data->timeout.secs ==INT_MAX ) { + thread->timeout =0 ; + }else { + thread->events |=DM_EVENT_TIMEOUT ; + thread->timeout = message_data->timeout.secs; + ret =_register_for_timeout (thread); + } + }_unlock_mutex (); -return thread ?0 : -ENODEV ; +return (thread && ret ==0 ) ?0 : -ENODEV ; }static int _get_timeout (struct message_data *message_data)
daemons/dmeventd/libdevmapper-event.c +76 -13
103 lines not on this page · raw message
daemons/dmeventd/libdevmapper-event.h +2 -0
@@ -102,5 +102,7 @@ void process_event(struct dm_task *dmt,int register_device (const char *device_name,const char *uuid,int major,int minor,void **user);int unregister_device (const char *device_name,const char *uuid,int major,int minor,void **user); +int dm_event_set_timeout (const char *dev_name,uint32_t timeout); +int dm_event_unset_timeout (const char *dev_name);#endif
PATCH 1/1 Add dm_event_set_timeout/dm_event_unset_timeout interface
Quoted 1 line · Malahal Naineni [malahal at us.ibm.com]
Will be used in mirror DSO to handle transient device failures.
Alasdair suggested to use -1 instead instead of INT_MAX to unregister a timeout. This patch supersedes the previous patch.
==========================================================
Add dm_event_set_timeout/dm_event_unset_timeout interface
Will be used in mirror DSO to handle transient device failures.
daemons/dmeventd/.exported_symbols +2 -0
@@ -17,3 +17,5 @@ dm_event_unregister_handler dm_event_get_registered_device dm_event_handler_set_timeout dm_event_handler_get_timeout +dm_event_set_timeout +dm_event_unset_timeout
daemons/dmeventd/dmeventd.c +19 -6
@@ -155,7 +155,7 @@ struct message_data { } events;union {char *str; -uint32_t secs; +int32_t secs; } timeout;struct dm_event_daemon_message *msg;/* Pointer to message buffer. */ };
@@ -187,7 +187,7 @@ struct thread_status {enum dm_event_mask current_events;/* bitfield for occured events. */ struct dm_task *current_task;time_t next_time; -uint32_t timeout; +int32_t timeout;struct dm_list timeout_list;void *dso_private;/* dso per-thread status variable */ };
@@ -363,7 +363,7 @@ static int _parse_message(struct message message_data->events.field = i; }if (message_data->timeout.str) { -uint32_t secs =atoi (message_data->timeout.str); +int32_t secs =atoi (message_data->timeout.str);dm_free (message_data->timeout.str); message_data->timeout.secs = secs ? secs :DM_EVENT_DEFAULT_TIMEOUT ;
@@ -1164,13 +1164,26 @@ static int _get_next_registered_device(sstatic int _set_timeout (struct message_data *message_data) {struct thread_status *thread; +int ret =0 ;_lock_mutex (); -if ((thread =_lookup_thread_status (message_data))) - thread->timeout = message_data->timeout.secs; +if ((thread =_lookup_thread_status (message_data))) { +/* A timeout of -1 (actually any negative timeout) is +* used to unregister the timeout, register a timeout +* for any other value. */ + thread->events &= ~DM_EVENT_TIMEOUT ; +_unregister_for_timeout (thread); +if (message_data->timeout.secs >=0 ) { + thread->events |=DM_EVENT_TIMEOUT ; + thread->timeout = message_data->timeout.secs; + ret =_register_for_timeout (thread); + }else { + thread->timeout =0 ; + } + }_unlock_mutex (); -return thread ?0 : -ENODEV ; +return (thread && ret ==0 ) ?0 : -ENODEV ; }static int _get_timeout (struct message_data *message_data)
daemons/dmeventd/libdevmapper-event.c +79 -17
131 lines not on this page · raw message
daemons/dmeventd/libdevmapper-event.h +2 -0
@@ -102,5 +102,7 @@ void process_event(struct dm_task *dmt,int register_device (const char *device_name,const char *uuid,int major,int minor,void **user);int unregister_device (const char *device_name,const char *uuid,int major,int minor,void **user); +int dm_event_set_timeout (const char *dev_name,int32_t timeout); +int dm_event_unset_timeout (const char *dev_name);#endif
PATCH 1/1 Add dm_event_set_timeout/dm_event_unset_timeout interface
Quoted 2 lines · malahal at us.ibm.com
Alasdair suggested to use -1 instead instead of
INT_MAXto unregister a timeout. This patch supersedes the previous patch.
Using a #define, please, rather than hard-coding -1 into the source.
Alasdair
PATCH 1/1 Add dm_event_set_timeout/dm_event_unset_timeout interface
Quoted 1 line · Alasdair G Kergon [agk at redhat.com]
Using a #define, please, rather than hard-coding -1 into the source.
Thank you. The following patch introduces DM_EVENT_REMOVE_TIMEOUT for that purpose. This patch supersedes the previous patch (both patches are against same source, so interdiff can be used to find just the difference)
=============================================================================
Add dm_event_set_timeout/dm_event_unset_timeout interface
Will be used in mirror DSO to handle transient device failures.
daemons/dmeventd/.exported_symbols +2 -0
@@ -17,3 +17,5 @@ dm_event_unregister_handler dm_event_get_registered_device dm_event_handler_set_timeout dm_event_handler_get_timeout +dm_event_set_timeout +dm_event_unset_timeout
daemons/dmeventd/dmeventd.c +19 -6
@@ -155,7 +155,7 @@ struct message_data { } events;union {char *str; -uint32_t secs; +int32_t secs; } timeout;struct dm_event_daemon_message *msg;/* Pointer to message buffer. */ };
@@ -187,7 +187,7 @@ struct thread_status {enum dm_event_mask current_events;/* bitfield for occured events. */ struct dm_task *current_task;time_t next_time; -uint32_t timeout; +int32_t timeout;struct dm_list timeout_list;void *dso_private;/* dso per-thread status variable */ };
@@ -363,7 +363,7 @@ static int _parse_message(struct message message_data->events.field = i; }if (message_data->timeout.str) { -uint32_t secs =atoi (message_data->timeout.str); +int32_t secs =atoi (message_data->timeout.str);dm_free (message_data->timeout.str); message_data->timeout.secs = secs ? secs :DM_EVENT_DEFAULT_TIMEOUT ;
@@ -1164,13 +1164,26 @@ static int _get_next_registered_device(sstatic int _set_timeout (struct message_data *message_data) {struct thread_status *thread; +int ret =0 ;_lock_mutex (); -if ((thread =_lookup_thread_status (message_data))) - thread->timeout = message_data->timeout.secs; +if ((thread =_lookup_thread_status (message_data))) { +/* A timeout of DM_EVENT_REMOVE_TIMEOUT is used to +* unregister the timeout, register a timeout for any +* other value. */ + thread->events &= ~DM_EVENT_TIMEOUT ; +_unregister_for_timeout (thread); +if (message_data->timeout.secs !=DM_EVENT_REMOVE_TIMEOUT ) { + thread->events |=DM_EVENT_TIMEOUT ; + thread->timeout = message_data->timeout.secs; + ret =_register_for_timeout (thread); + }else { + thread->timeout =0 ; + } + }_unlock_mutex (); -return thread ?0 : -ENODEV ; +return (thread && ret ==0 ) ?0 : -ENODEV ; }static int _get_timeout (struct message_data *message_data)
daemons/dmeventd/dmeventd.h +1 -0
@@ -24,6 +24,7 @@#define DM_EVENT_PIDFILE "/var/run/dmeventd.pid"#define DM_EVENT_DEFAULT_TIMEOUT 10 +#define DM_EVENT_REMOVE_TIMEOUT -1/* used to unschedule a timeout */ /* Commands for the daemon passed in the message below. */ enum dm_event_command {
daemons/dmeventd/libdevmapper-event.c +79 -17
131 lines not on this page · raw message
daemons/dmeventd/libdevmapper-event.h +2 -0
@@ -102,5 +102,7 @@ void process_event(struct dm_task *dmt,int register_device (const char *device_name,const char *uuid,int major,int minor,void **user);int unregister_device (const char *device_name,const char *uuid,int major,int minor,void **user); +int dm_event_set_timeout (const char *dev_name,int32_t timeout); +int dm_event_unset_timeout (const char *dev_name);#endif
PATCH 1/1 Attempt to resync a failed secondary leg few times before giving up
In reply to Malahal NaineniThis patch adds the capability to attempt resync of a failed mirror device at a given timeout interval and given number of attempts.
daemons/dmeventd/plugins/mirror/Makefile.in +1 -1
@@ -32,7 +32,7 @@ LIB_VERSION = $(LIB_VERSION_LVM) include $(top_builddir)/make.tmpl -LIBS += -ldevmapper @LIB_PTHREAD@ @LVM2CMD_LIB@ +LIBS += -ldevmapper -ldevmapper-event @LIB_PTHREAD@ @LVM2CMD_LIB@ install_lvm2: libdevmapper-event-lvm2mirror.$(LIB_SUFFIX) $(INSTALL) -D $(OWNER) $(GROUP) -m 555 $(STRIP) $< \
daemons/dmeventd/plugins/mirror/dmeventd_mirror.c +84 -7
160 lines not on this page · raw message