Waiting for the missing device in mirror

index 1251a5d..e955a02 100644
@@ -7036,6 +7036,7 @@ static struct logical_volume *_lv_create_an_lv(struct volume_group *vg,
 
 		/* FIXME This will not pass cluster lock! */
 		init_mirror_in_sync(lp->nosync);
+		init_track_changes(lp->track_changes);
 
 		if (lp->nosync) {
 			log_warn("WARNING: New %s won't be synchronised. "

index 0e52153..7b5e078 100644
@@ -848,6 +848,7 @@ struct lvcreate_params {
 	int32_t minor; /* all */
 	int log_count; /* mirror */
 	int nosync; /* mirror */
+	int track_changes; /* mirror */
 	int pool_metadata_spare; /* pools */
 	int type;   /* type arg is given */
 	int temporary; /* temporary LV */

index e57e9bb..8447b8e 100644
@@ -371,6 +371,9 @@ static int _add_log(struct dm_pool *mem, struct lv_segment *seg,
 	if (_block_on_error_available && !(seg->status & PVMOVE))
 		log_flags |= DM_BLOCK_ON_ERROR;
 
+	if (track_changes())
+		log_flags |= DM_KEEPLOG;
+
 	return dm_tree_node_add_mirror_target_log(node, region_size, clustered, log_dlid, area_count, log_flags);
 }
 

index 6455788..3a00beb 100644
@@ -39,6 +39,7 @@ static int _ignorelockingfailure = 0;
 static int _security_level = SECURITY_LEVEL;
 static char _cmd_name[30] = "";
 static int _mirror_in_sync = 0;
+static int _track_changes = 0;
 static int _dmeventd_monitor = DEFAULT_DMEVENTD_MONITOR;
 static int _background_polling = DEFAULT_BACKGROUND_POLLING;
 static int _ignore_suspended_devices = 0;

@@ -121,6 +122,11 @@ void init_mirror_in_sync(int in_sync)
 	_mirror_in_sync = in_sync;
 }
 
+void init_track_changes(int track_changes)
+{
+	_track_changes = track_changes;
+}
+
 void init_dmeventd_monitor(int reg)
 {
 	_dmeventd_monitor = reg;

@@ -277,6 +283,11 @@ int mirror_in_sync(void)
 	return _mirror_in_sync;
 }
 
+int track_changes(void)
+{
+	return _track_changes;
+}
+
 int dmeventd_monitor_mode(void)
 {
 	return _dmeventd_monitor;

index b25f4ae..55ff3c2 100644
@@ -37,6 +37,7 @@ void init_ignorelockingfailure(int level);
 void init_lockingfailed(int level);
 void init_security_level(int level);
 void init_mirror_in_sync(int in_sync);
+void init_track_changes(int track_changes);
 void init_dmeventd_monitor(int reg);
 void init_background_polling(int polling);
 void init_ignore_suspended_devices(int ignore);

@@ -69,6 +70,7 @@ int ignorelockingfailure(void);
 int lockingfailed(void);
 int security_level(void);
 int mirror_in_sync(void);
+int track_changes(void);
 int background_polling(void);
 int ignore_suspended_devices(void);
 int ignore_lvm_mirrors(void);

index c811641..30cab21 100644
@@ -717,6 +717,7 @@ int dm_tree_node_add_mirror_target(struct dm_tree_node *node,
 #define DM_FORCESYNC		0x00000002	/* Force resync */
 #define DM_BLOCK_ON_ERROR	0x00000004	/* On error, suspend I/O */
 #define DM_CORELOG		0x00000008	/* In-memory log */
+#define DM_KEEPLOG      0x00000010  /* Keep log while device missing*/
 
 int dm_tree_node_add_mirror_target_log(struct dm_tree_node *node,
 				       uint32_t region_size,

index 578f645..c08c776 100644
@@ -2167,6 +2167,7 @@ static int _mirror_emit_segment_line(struct dm_task *dmt, struct load_segment *s
 	int block_on_error = 0;
 	int handle_errors = 0;
 	int dm_log_userspace = 0;
+	int track_changes = 0;
 	struct utsname uts;
 	unsigned log_parm_count;
 	int pos = 0, parts;

@@ -2203,6 +2204,14 @@ static int _mirror_emit_segment_line(struct dm_task *dmt, struct load_segment *s
 			block_on_error = 1;
 	}
 
+	if (seg->flags & DM_KEEPLOG) {
+		track_changes = 1;
+		if (!handle_errors) {
+			log_error("Feature keep_log depends on handle_errors");
+			return_0;
+		}
+	}
+
 	if (seg->clustered) {
 		/* Cluster mirrors require a UUID */
 		if (!seg->uuid)

@@ -2229,6 +2238,10 @@ static int _mirror_emit_segment_line(struct dm_task *dmt, struct load_segment *s
 	/* "handle_errors" is a feature arg now */
 	if (handle_errors)
 		log_parm_count--;
+	
+	if (track_changes)
+		log_parm_count--;
+	
 
 	/* DM_CORELOG does not count in the param list */
 	if (seg->flags & DM_CORELOG)

@@ -2280,7 +2293,9 @@ static int _mirror_emit_segment_line(struct dm_task *dmt, struct load_segment *s
 	if (_emit_areas_line(dmt, seg, params, paramsize, &pos) <= 0)
 		return_0;
 
-	if (handle_errors)
+	if (handle_errors && track_changes)
+		EMIT_PARAMS(pos, " 2 handle_errors keep_log");
+	else if (handle_errors)
 		EMIT_PARAMS(pos, " 1 handle_errors");
 
 	return 1;

index 8e87681..003fc57 100644
@@ -315,7 +315,7 @@ xx(lvcreate,
    "\t -L|--size LogicalVolumeSize[bBsSkKmMgGtTpPeE]}\n"
    "\t[-M|--persistent {y|n}] [-j|--major major] [--minor minor]\n"
    "\t[--metadataprofile ProfileName]\n"
-   "\t[-m|--mirrors Mirrors [--nosync]\n"
+   "\t[-m|--mirrors Mirrors [--nosync][--trackchanges]\n"
    "\t  [{--mirrorlog {disk|core|mirrored}|--corelog}]]\n"
    "\t[-n|--name LogicalVolumeName]\n"
    "\t[--noudevsync]\n"

@@ -381,7 +381,7 @@ xx(lvcreate,
    chunksize_ARG, contiguous_ARG, corelog_ARG, discards_ARG, errorwhenfull_ARG,
    extents_ARG, ignoreactivationskip_ARG, ignoremonitoring_ARG, major_ARG,
    metadataprofile_ARG, minor_ARG, mirrorlog_ARG, mirrors_ARG, monitor_ARG,
-   minrecoveryrate_ARG, maxrecoveryrate_ARG, name_ARG, nosync_ARG,
+   minrecoveryrate_ARG, maxrecoveryrate_ARG, name_ARG, nosync_ARG, trackchanges_ARG,
    noudevsync_ARG, permission_ARG, persistent_ARG,
    //pooldatasize_ARG,
    poolmetadatasize_ARG, poolmetadataspare_ARG,

index e41f76c..ac64677 100644
@@ -535,6 +535,7 @@ static int _read_mirror_and_raid_params(struct cmd_context *cmd,
 		lp->mirrors = seg_is_mirrored(lp) ? 2 : 1;
 
 	lp->nosync = arg_is_set(cmd, nosync_ARG);
+	lp->track_changes = arg_is_set(cmd, trackchanges_ARG);
 
 	if (!(lp->region_size = arg_uint_value(cmd, regionsize_ARG, 0)) &&
 	    ((lp->region_size = get_default_region_size(cmd)) <= 0)) {

@@ -720,7 +721,8 @@ static int _lvcreate_params(struct cmd_context *cmd,
 
 #define MIRROR_ARGS \
 	corelog_ARG,\
-	mirrorlog_ARG
+	mirrorlog_ARG,\
+	trackchanges_ARG
 
 #define MIRROR_RAID_ARGS \
 	nosync_ARG,\

index fe8b761..8fcbefd 100644
@@ -962,6 +962,49 @@ static void _lvconvert_mirrors_repair_ask(struct cmd_context *cmd,
 }
 
 /*
+ * Here we use a tmp file named UUID of pv under /tmp
+ * to identify if this device already being waited.
+ */
+#define WAITING_DIR  "/tmp/"
+static int already_being_waited(struct physical_volume *pv)
+{
+	char uuid[64];
+	char path[70];
+	FILE *fp;
+	int r = 0;
+
+	id_write_format(&pv->id, uuid, 64);
+	sprintf(path, WAITING_DIR"%s", uuid);
+	fp = fopen(path, "r");
+	if (!fp) {
+		fp = fopen(path, "w");
+		fclose(fp);
+	} else
+		r = 1;
+	return r;
+}
+
+static int remove_device_from_waiting_dir(struct physical_volume *pv)
+{
+	char uuid[64];
+	char path[70];
+	FILE *fp;
+	int r = 1;
+
+	id_write_format(&pv->id, uuid, 64);
+	sprintf(path, WAITING_DIR"%s", uuid);
+	fp = fopen(path, "r");
+	if (!fp) {
+		log_error("It should open the file successfully");
+		r = 0;
+	} else {
+		fclose(fp);
+		remove(path);
+	}
+	return r;
+}
+
+/*
  * _get_log_count
  * @lv: the mirror LV
  *

index 018772e..be7f06d 100644
@@ -504,6 +504,10 @@ static int _print_pvs(struct formatter *f, struct volume_group *vg)
 
 		outhint(f, "device = \"%s\"",
 			dm_escape_double_quotes(buffer, pv_dev_name(pv)));
+		if (pv_dev(pv))
+			outhint(f, "dev_t = %u", pv_dev(pv)->dev);
+		else
+			outhint(f, "dev_t = %u", pv->old_dev);
 		outnl(f);
 
 		if (!_print_flag_config(f, pv->status, PV_FLAGS))

index 2d62905..f33c01e 100644
@@ -228,6 +228,11 @@ static int _read_pv(struct format_instance *fid,
 		log_info("Recovering a previously MISSING PV %s with no MDAs.",
 			 pv_dev_name(pv));
 	}
+	if ((pv->status & MISSING_PV) && dm_config_has_node(pvn, "dev_t") &&
+			!_read_uint32(pvn, "dev_t", &pv->old_dev)) {
+		log_error("Couldn't read dev_t for physical volume when it's missing.");
+		return 0;
+	}
 
 	/* Late addition */
 	if (dm_config_has_node(pvn, "dev_size") &&

index 2d436cb..cdd0674 100644
@@ -26,6 +26,7 @@ struct physical_volume {
 	struct id id;
 	struct id old_id;		/* Set during pvchange -u. */
 	struct device *dev;
+	dev_t old_dev;          /*Only meaningfull when pv is missing*/
 	const struct format_type *fmt;
 	struct format_instance *fid;
 

index 885a2fa..bfea715 100644
@@ -1147,6 +1147,8 @@ cfg(activation_mirror_log_fault_policy_CFG, "mirror_log_fault_policy", activatio
 	"Defines how a device failure in a 'mirror' log LV is handled.\n"
 	"The mirror_image_fault_policy description for mirrored LVs\n"
 	"also applies to mirrored log LVs.\n")
+cfg(activation_mirror_keep_log_CFG, "mirror_keep_log", activation_CFG_SECTION, 0, CFG_TYPE_INT, DEFAULT_MIRROR_KEEP_LOG, vsn(4, 2, 0), NULL,0, NULL, "Enable to wait for the missing device come back within a fixed time.\n")
+cfg(activation_mirror_keep_log_timeout_CFG, "mirror_keep_log_timeout", activation_CFG_SECTION, 0, CFG_TYPE_INT, DEFAULT_MIRROR_KEEP_LOG_TIMEOUT, vsn(4, 2, 0), NULL, 0, NULL, "Defines the wating time.\n")
 
 cfg(activation_mirror_device_fault_policy_CFG, "mirror_device_fault_policy", activation_CFG_SECTION, 0, CFG_TYPE_STRING, DEFAULT_MIRROR_DEVICE_FAULT_POLICY, vsn(1, 2, 10), NULL, vsn(2, 2, 57),
 	"This has been replaced by the activation/mirror_image_fault_policy setting.\n",

index 6793d01..6fbadfd 100644
@@ -216,6 +216,8 @@
 
 #define DEFAULT_MIRROR_DEVICE_FAULT_POLICY "remove"
 #define DEFAULT_MIRROR_LOG_FAULT_POLICY "allocate"
+#define DEFAULT_MIRROR_KEEP_LOG 0
+#define DEFAULT_MIRROR_KEEP_LOG_TIMEOUT 60
 #define DEFAULT_SNAPSHOT_AUTOEXTEND_THRESHOLD 100
 #define DEFAULT_SNAPSHOT_AUTOEXTEND_PERCENT 20
 #define DEFAULT_THIN_POOL_AUTOEXTEND_THRESHOLD 100

index 8fcbefd..47564de 100644
@@ -16,6 +16,9 @@
 #include "polldaemon.h"
 #include "lv_alloc.h"
 #include "lvconvert_poll.h"
+#include "lvm-wrappers.h"
+#include <blkid/blkid.h>
+#include <libudev.h>
 
 struct lvconvert_params {
 	int cache;

@@ -925,6 +928,13 @@ static struct logical_volume *_original_lv(struct logical_volume *lv)
 	return next_lv;
 }
 
+/*
+ * replace_log/replace_mirrors value
+ * 0: remove
+ * 1: replace
+ * 2: wait+remove
+ * 3: wait+replace
+ */
 static void _lvconvert_mirrors_repair_ask(struct cmd_context *cmd,
 					  int failed_log, int failed_mirrors,
 					  int *replace_log, int *replace_mirrors)

@@ -932,12 +942,20 @@ static void _lvconvert_mirrors_repair_ask(struct cmd_context *cmd,
 	const char *leg_policy, *log_policy;
 	int force = arg_count(cmd, force_ARG);
 	int yes = arg_count(cmd, yes_ARG);
+	int keep_log = 0;
 
 	if (arg_count(cmd, use_policies_ARG)) {
 		leg_policy = find_config_tree_str(cmd, activation_mirror_image_fault_policy_CFG, NULL);
 		log_policy = find_config_tree_str(cmd, activation_mirror_log_fault_policy_CFG, NULL);
 		*replace_mirrors = strcmp(leg_policy, "remove");
 		*replace_log = strcmp(log_policy, "remove");
+		keep_log = find_config_tree_int(cmd, activation_mirror_keep_log_CFG, NULL);
+		*replace_mirrors = !!strcmp(leg_policy, "remove");
+		*replace_log = !!strcmp(log_policy, "remove");
+		if (keep_log) {
+			*replace_mirrors += 2;
+			*replace_log += 2;
+		}
 		return;
 	}
 

@@ -1004,6 +1022,193 @@ static int remove_device_from_waiting_dir(struct physical_volume *pv)
 	return r;
 }
 
+static int get_a_missing_device(struct logical_volume *lv, struct physical_volume **pv)
+{
+	struct lv_segment *lvseg;
+	unsigned s;
+	int r = 0;
+
+
+	dm_list_iterate_items(lvseg, &lv->segments) {
+		for (s = 0; s < lvseg->area_count; s++) {
+			if (seg_type(lvseg, s) == AREA_LV) {
+				if (is_temporary_mirror_layer(seg_lv(lvseg, s)))
+					return get_a_missing_device(seg_lv(lvseg, s), pv);
+				else if (seg_lv(lvseg, s)->status & PARTIAL_LV)
+					return get_a_missing_device(seg_lv(lvseg, s), pv);
+			}
+			else if (seg_type(lvseg, s) == AREA_PV &&
+				 is_missing_pv(seg_pv(lvseg, s))) {
+				if (already_being_waited(seg_pv(lvseg, s)))
+						continue;
+				*pv = seg_pv(lvseg, s);
+				r = 1;
+				break;
+			}
+		}
+		if (r == 1)
+			break;
+	}
+	return r;
+}
+
+static int missing_device_return(struct logical_volume *lv, struct physical_volume **pv,
+	            	dev_t *old, dev_t *new, int wait_time)
+{
+	struct udev_monitor *mon;
+	struct udev_device *device;
+	const char *new_uuid, *dev_nod;
+	struct udev *udev;
+	char uuid[64] __attribute__((aligned(8)));
+	blkid_probe pr;
+	int fd;
+	fd_set fds;
+	struct timeval tv;
+	int r = 0;
+
+	r = get_a_missing_device(lv, pv);
+	if (!r) {
+		log_error("Failed to get the missing pv");
+		goto bad;
+	}
+	/*
+	 * How to get the missing device dev_t?
+	 */
+	*old = (*pv)->old_dev;
+	if (!*old) {
+		log_error("Failed to get the dev_t of missing pv");
+		goto bad;
+	}
+
+	if (!id_write_format((&(*pv)->id), uuid, sizeof(uuid))) {
+		stack;
+		r = 0;
+		goto bad;
+	}
+
+	if (!(udev = udev_new())) {
+		r = 0;
+		goto bad;
+	}
+
+	mon = udev_monitor_new_from_netlink(udev, "udev");
+	udev_monitor_filter_add_match_subsystem_devtype(mon, "block", NULL);
+	udev_monitor_enable_receiving(mon);
+	fd = udev_monitor_get_fd(mon);
+
+	tv.tv_sec = wait_time;
+	tv.tv_usec = 0;
+	FD_ZERO(&fds);
+	FD_SET(fd, &fds);
+	while(1) {
+		r = select(fd+1, &fds, NULL, NULL, &tv);
+		if (r > 0 && FD_ISSET(fd, &fds)) {
+			/*
+			 * check if it's the missing device
+			 */
+			device = udev_monitor_receive_device(mon);
+			if (device) {
+				dev_nod = udev_device_get_devnode(device);
+				if (NULL == dev_nod) {
+					continue;
+				}
+				pr = blkid_new_probe_from_filename(dev_nod);
+				if (!pr) {
+					log_error("Failed to open %s",dev_nod);
+					goto out;
+				}
+				blkid_do_probe(pr);
+				blkid_probe_lookup_value(pr, "UUID", &new_uuid, NULL);
+
+				if (!strncmp(new_uuid, uuid, sizeof(uuid))) {
+					*new = udev_device_get_devnum(device);
+					r = 1;
+					break;
+				}
+				blkid_free_probe(pr);
+				udev_device_unref(device);
+			} else {
+				log_error("No device from udev_monitor_receive_device.");
+				goto out;
+			}
+		}
+		/*
+		 * timeout while waiting the missing device
+		 */
+		else if (r == 0) {
+			break;
+		}
+	}
+
+out:
+	udev_unref(udev);
+bad:
+	if(*pv)
+		remove_device_from_waiting_dir(*pv);
+	return r;
+}
+
+static int start_sync_lv(struct logical_volume *lv)
+{
+	char target[70];
+	int r = 1;
+
+	sprintf(target,"%s-%s", lv->vg->name, lv->name);
+
+	if (!dm_lv_simple_suspend(target) ||
+			!dm_lv_simple_resume(target)) {
+		r = 0;
+		log_error("Failed to suspend/resume %s", lv->name);
+	}
+
+	return r;
+}
+
+static int lv_replace_table(struct cmd_context *cmd, struct logical_volume *lv,
+		         struct physical_volume *pv, dev_t old, dev_t new)
+{
+	struct logical_volume *sub_lv = NULL;
+	struct lv_segment *lvseg;
+	char target[70];
+	unsigned s;
+	int r = 0;
+
+	/*check which lv is on the missing pv*/
+	dm_list_iterate_items(lvseg, &lv->segments) {
+		for (s = 0; s < lvseg->area_count; s++) {
+			if ((seg_type(lvseg, s) == AREA_LV) &&
+			     (seg_lv(lvseg, s)->status & PARTIAL_LV)) {
+				sub_lv = seg_lv(lvseg, s);
+				if (lv_is_on_pv(sub_lv, pv)) {
+					r = 1;
+					break;
+				}
+			}
+		}
+		if (1 == r) {
+			r = 0;
+			break;
+		}
+	}
+
+	if (!sub_lv)
+		goto fail;
+
+	sprintf(target,"%s-%s", sub_lv->vg->name, sub_lv->name);
+	if (!dm_lv_simple_suspend(target))
+		goto fail;
+
+	if (!dm_replace_mirror_table(target, old, new))
+		goto fail;
+
+	if (!dm_lv_simple_resume(target))
+		goto fail;
+	r = 1;
+fail:
+	return r;
+}
+
+
 /*
  * _get_log_count
  * @lv: the mirror LV

@@ -1510,6 +1715,10 @@ static int _lvconvert_mirrors_repair(struct cmd_context *cmd,
 	int replace_logs = 0;
 	int replace_mimages = 0;
 	uint32_t log_count;
+	int wait_time = 0;
+	int daemon_mode = 0;
+	dev_t old, new;
+	struct physical_volume *pv = NULL;
 
 	uint32_t original_mimages = lv_mirror_count(lv);
 	uint32_t original_logs = _get_log_count(lv);

@@ -1525,28 +1734,79 @@ static int _lvconvert_mirrors_repair(struct cmd_context *cmd,
 		return 1;
 	}
 
+	/*
+	 * Count the failed log devices
+	 */
 	failed_mimages = _failed_mirrors_count(lv);
 	failed_logs = _failed_logs_count(lv);
 
-	if (!mirror_remove_missing(cmd, lv, 0))
-		return_0;
+	/*
+	 * Find out our policies
+	 */
+	_lvconvert_mirrors_repair_ask(cmd, failed_logs, failed_mimages,
+					  &replace_logs, &replace_mimages);
+
 
 	if (failed_mimages)
 		log_print_unless_silent("Mirror status: %d of %d images failed.",
 					failed_mimages, original_mimages);
 
-	/*
-	 * Count the failed log devices
-	 */
-	if (failed_logs)
+	if (failed_logs) {
 		log_print_unless_silent("Mirror log status: %d of %d images failed.",
 					failed_logs, original_logs);
+		if (replace_logs >= 2) {
+			log_error("The log device failed, there is probably no meaning to keep the bitmap."
+					"We make keep_log doesn't work on log devices");
+			replace_logs -= 2;
+		}
+	}
 
-	/*
-	 * Find out our policies
-	 */
-	_lvconvert_mirrors_repair_ask(cmd, failed_logs, failed_mimages,
-				      &replace_logs, &replace_mimages);
+	if (replace_mimages >= 2) {
+		if (!udev_is_running()) {
+			log_error("Udev is not running!This feature need support by udev");
+			goto feature_fail;
+		}
+		/*
+		 *  Here we make it a daemon to wait
+		 */
+		daemon_mode = become_daemon(cmd, 0);
+		if (daemon_mode == 0)
+			return ECMD_PROCESSED; /*CHECK: safe for parent to return directly*/
+		else if (daemon_mode != 1) {
+			log_error("Failed to become daemon and disable this feature");
+			goto feature_fail;
+		}
+		wait_time = find_config_tree_int(cmd, activation_mirror_keep_log_timeout_CFG, NULL);
+		log_print_unless_silent("This mirror LV will wait %d secs before"
+				"adopting new policy", wait_time);
+		if (missing_device_return(lv, &pv, &old, &new, wait_time)) {
+			/* Remove the MISSING flag*/
+			pv->status &= ~MISSING_PV;
+	
+			if (!pv) {
+				log_error("Failed to get the missing device");
+				goto feature_fail;
+			}
+			if ((MAJOR(old) == MAJOR(new)) && (MINOR(old) == MINOR(new)))
+				goto skip_replace;
+			if (!lv_replace_table(cmd, lv, pv, old, new)) {
+				log_error("Failed to replace table since major/minor changed");
+				goto feature_fail;
+			}
+skip_replace:
+			/*Do sync now*/
+			if (!start_sync_lv(lv)) {
+				log_error("Failed to start do the inremental sync");
+				goto feature_fail;
+			}
+			return 1;
+		}
+feature_fail:
+		replace_mimages -= 2;
+	}
+
+	if (!mirror_remove_missing(cmd, lv, 0))
+		return_0;
 
 	/*
 	 * Second phase - replace faulty devices