lvm2: mirroredlog support

This patch adds '--mirroredlog' option to LVM commands to create a mirror with mirrored log device. Rebased to the latest LVM code (LVM2.2.02.43). Appreciate any comments.

---
 lib/activate/activate.c          |   16 ++++++
 lib/metadata/metadata-exported.h |   17 +++++--
 lib/metadata/mirror.c            |   90 ++++++++++++++++++++++++++++-----------
 tools/args.h                     |    1
 tools/commands.h                 |    7 ++-
 tools/lvconvert.c                |   17 +++----
 tools/lvcreate.c                 |    9 +++
 tools/pvmove.c                   |    4 -
 8 files changed, 118 insertions(+), 43 deletions(-)

diff -r d548a82e6195 lib/activate/activate.c
@@ -948,6 +948,8 @@
 {
 	struct logical_volume *lv;
 	struct lvinfo info;
+	struct lv_segment *seg, *log_seg;
+	struct segment_type *type = get_segtype_from_string(cmd, "mirror");
 	int r;
 
 	if (!activation())

@@ -975,6 +977,20 @@
 
 	if (!monitor_dev_for_events(cmd, lv, 0))
 		stack;
+
+	if ( !dm_list_empty(&lv->segments) ) {
+		seg = dm_list_item(dm_list_first(&lv->segments), struct lv_segment);
+		if (seg->log_lv) {
+			log_very_verbose("lv %s is mirrored, check it's log %s...", lv->name, seg->log_lv->name);
+			if ( !dm_list_empty(&seg->log_lv->segments) ) {
+				log_seg = dm_list_item(dm_list_first(&seg->log_lv->segments), struct lv_segment);
+				if (log_seg->segtype == type) {
+					log_verbose("log %s is mirrored, unregister for events", log_seg->lv->name);
+					monitor_dev_for_events(cmd, log_seg->lv, 0);
+				}
+			}
+		}
+	}
 
 	memlock_inc();
 	r = _lv_deactivate(lv);

diff -r d548a82e6195 lib/metadata/metadata-exported.h
@@ -505,9 +505,11 @@
 */
 struct lv_segment *find_mirror_seg(struct lv_segment *seg);
 int lv_add_mirrors(struct cmd_context *cmd, struct logical_volume *lv,
-		   uint32_t mirrors, uint32_t stripes,
+		   const struct segment_type *segtype,
+		   uint32_t mirrors, uint32_t stripes, uint32_t stripe_size,
 		   uint32_t region_size, uint32_t log_count,
-		   struct dm_list *pvs, alloc_policy_t alloc, uint32_t flags);
+		   struct dm_list *pvs, alloc_policy_t alloc, uint32_t flags,
+		   uint32_t mirrored_log);
 int lv_remove_mirrors(struct cmd_context *cmd, struct logical_volume *lv,
 		      uint32_t mirrors, uint32_t log_count,
 		      struct dm_list *pvs, uint32_t status_mask);

@@ -526,16 +528,21 @@
 int remove_mirror_images(struct logical_volume *lv, uint32_t num_mirrors,
 			 struct dm_list *removable_pvs, unsigned remove_log);
 int add_mirror_images(struct cmd_context *cmd, struct logical_volume *lv,
-		      uint32_t mirrors, uint32_t stripes, uint32_t region_size,
+		      const struct segment_type *segtype,
+		      uint32_t mirrors, uint32_t stripes, uint32_t stripe_size,
+		      uint32_t region_size,
 		      struct dm_list *allocatable_pvs, alloc_policy_t alloc,
-		      uint32_t log_count);
+		      uint32_t log_count, uint32_t mirrored_log);
 struct logical_volume *detach_mirror_log(struct lv_segment *seg);
 int attach_mirror_log(struct lv_segment *seg, struct logical_volume *lv);
 int remove_mirror_log(struct cmd_context *cmd, struct logical_volume *lv,
 		      struct dm_list *removable_pvs);
 int add_mirror_log(struct cmd_context *cmd, struct logical_volume *lv,
+		   const struct segment_type *segtype,
 		   uint32_t log_count, uint32_t region_size,
-		   struct dm_list *allocatable_pvs, alloc_policy_t alloc);
+		   struct dm_list *allocatable_pvs, alloc_policy_t alloc,
+		   uint32_t mirrors, uint32_t stripes, uint32_t stripe_size,
+		   uint32_t mirrored_log);
 
 int reconfigure_mirror_images(struct lv_segment *mirrored_seg, uint32_t num_mirrors,
 			      struct dm_list *removable_pvs, unsigned remove_log);

diff -r d548a82e6195 tools/args.h
@@ -45,6 +45,7 @@
 arg(alloc_ARG, '\0', "alloc", alloc_arg, 0)
 arg(separator_ARG, '\0', "separator", string_arg, 0)
 arg(mirrorsonly_ARG, '\0', "mirrorsonly", NULL, 0)
+arg(mirroredlog_ARG, '\0', "mirroredlog", NULL, 0)
 arg(nosync_ARG, '\0', "nosync", NULL, 0)
 arg(resync_ARG, '\0', "resync", NULL, 0)
 arg(corelog_ARG, '\0', "corelog", NULL, 0)

diff -r d548a82e6195 tools/commands.h
@@ -93,7 +93,7 @@
    "Change logical volume layout",
    0,
    "lvconvert "
-   "[-m|--mirrors Mirrors [{--mirrorlog {disk|core}|--corelog}]]\n"
+   "[-m|--mirrors Mirrors [{--mirrorlog {disk|core}|--corelog|--mirroredlog}]]\n"
    "\t[-R|--regionsize MirrorLogRegionSize]\n"
    "\t[--alloc AllocationPolicy]\n"
    "\t[-b|--background]\n"

@@ -115,7 +115,8 @@
    "\tOriginalLogicalVolume[Path] SnapshotLogicalVolume[Path]\n",
 
    alloc_ARG, background_ARG, chunksize_ARG, corelog_ARG, interval_ARG,
-   mirrorlog_ARG, mirrors_ARG, regionsize_ARG, snapshot_ARG, test_ARG, zero_ARG)
+   mirrorlog_ARG, mirrors_ARG, regionsize_ARG, mirroredlog_ARG,
+   snapshot_ARG, test_ARG, zero_ARG)
 
 xx(lvcreate,
    "Create a logical volume",

@@ -132,6 +133,7 @@
    "\t -L|--size LogicalVolumeSize[kKmMgGtTpPeE]}\n"
    "\t[-M|--persistent {y|n}] [--major major] [--minor minor]\n"
    "\t[-m|--mirrors Mirrors [--nosync] [{--mirrorlog {disk|core}|--corelog}]]\n"
+   "\t[--mirroredlog]\n"
    "\t[-n|--name LogicalVolumeName]\n"
    "\t[-p|--permission {r|rw}]\n"
    "\t[-r|--readahead ReadAheadSectors|auto|none]\n"

@@ -163,6 +165,7 @@
    "\t[--version]\n"
    "\tOriginalLogicalVolume[Path] [PhysicalVolumePath...]\n\n",
 
+   mirroredlog_ARG,
    addtag_ARG, alloc_ARG, autobackup_ARG, chunksize_ARG, contiguous_ARG,
    corelog_ARG, extents_ARG, major_ARG, minor_ARG, mirrorlog_ARG, mirrors_ARG,
    name_ARG, nosync_ARG, permission_ARG, persistent_ARG, readahead_ARG,

diff -r d548a82e6195 tools/lvconvert.c
@@ -383,6 +383,7 @@
 	const char *mirrorlog;
 	unsigned corelog = 0;
 	struct logical_volume *original_lv;
+	uint32_t mirrored_log = arg_count(cmd, mirroredlog_ARG);
 
 	seg = first_seg(lv);
 	existing_mirrors = lv_mirror_count(lv);

@@ -484,13 +485,13 @@
 			}
 		}
 
-		if (!lv_add_mirrors(cmd, lv, lp->mirrors - 1, 1,
+		if (!lv_add_mirrors(cmd, lv, lp->segtype, lp->mirrors - 1, 1, 0,
 				    adjusted_mirror_region_size(
 						lv->vg->extent_size,
 						lv->le_count,
 						lp->region_size),
 				    corelog ? 0U : 1U, lp->pvh, lp->alloc,
-				    MIRROR_BY_LV))
+				    MIRROR_BY_LV, mirrored_log))
 			return_0;
 		if (lp->wait_completion)
 			lp->need_polling = 1;

@@ -513,12 +514,12 @@
 		 */
 		original_lv = _original_lv(lv);
 		if (!first_seg(original_lv)->log_lv && !corelog) {
-			if (!add_mirror_log(cmd, original_lv, 1,
+			if (!add_mirror_log(cmd, original_lv, lp->segtype, 1,
 					    adjusted_mirror_region_size(
 							lv->vg->extent_size,
 							lv->le_count,
 							lp->region_size),
-					    lp->pvh, lp->alloc))
+					    lp->pvh, lp->alloc, lp->mirrors, 0, 0, mirrored_log))
 				return_0;
 		} else if (first_seg(original_lv)->log_lv && corelog) {
 			if (!remove_mirror_log(cmd, original_lv,

@@ -546,12 +547,12 @@
 		 */
 		original_lv = _original_lv(lv);
 		if (!first_seg(original_lv)->log_lv && !corelog) {
-			if (!add_mirror_log(cmd, original_lv, 1,
+			if (!add_mirror_log(cmd, original_lv, lp->segtype, 1,
 					    adjusted_mirror_region_size(
 							lv->vg->extent_size,
 							lv->le_count,
 							lp->region_size),
-					    lp->pvh, lp->alloc))
+					    lp->pvh, lp->alloc, lp->mirrors, 0, 0, mirrored_log))
 				return_0;
 		} else if (first_seg(original_lv)->log_lv && corelog) {
 			if (!remove_mirror_log(cmd, original_lv,

@@ -565,13 +566,13 @@
 			return 0;
 		}
 		/* FIXME: can't have multiple mlogs. force corelog. */
-		if (!lv_add_mirrors(cmd, lv, lp->mirrors - existing_mirrors, 1,
+		if (!lv_add_mirrors(cmd, lv, NULL, lp->mirrors - existing_mirrors, 1, 0,
 				    adjusted_mirror_region_size(
 						lv->vg->extent_size,
 						lv->le_count,
 						lp->region_size),
 				    0U, lp->pvh, lp->alloc,
-				    MIRROR_BY_LV))
+				    MIRROR_BY_LV, lp->mirrors))
 			return_0;
 		lv->status |= CONVERTING;
 		lp->need_polling = 1;

diff -r d548a82e6195 tools/lvcreate.c
@@ -37,6 +37,7 @@
 	uint32_t region_size;
 
 	uint32_t mirrors;
+	uint32_t mirrored_log;
 
 	const struct segment_type *segtype;
 

@@ -320,6 +321,9 @@
 		}
 		lp->region_size = region_size;
 	}
+
+	if (arg_count(cmd, mirroredlog_ARG))
+		lp->mirrored_log = 1;
 
 	if (!_validate_mirror_params(cmd, lp))
 		return 0;

@@ -772,14 +776,15 @@
 		return_0;
 
 	if (lp->mirrors > 1) {
-		if (!lv_add_mirrors(cmd, lv, lp->mirrors - 1, lp->stripes,
+		if (!lv_add_mirrors(cmd, lv, lp->segtype, lp->mirrors - 1, lp->stripes, lp->stripe_size,
 				    adjusted_mirror_region_size(
 						vg->extent_size,
 						lv->le_count,
 						lp->region_size),
 				    lp->corelog ? 0U : 1U, pvh, lp->alloc,
 				    MIRROR_BY_LV |
-				    (lp->nosync ? MIRROR_SKIP_INIT_SYNC : 0))) {
+				    (lp->nosync ? MIRROR_SKIP_INIT_SYNC : 0),
+				    lp->mirrored_log)) {
 			stack;
 			goto revert_new_lv;
 		}

diff -r d548a82e6195 tools/pvmove.c
@@ -251,8 +251,8 @@
 		return NULL;
 	}
 
-	if (!lv_add_mirrors(cmd, lv_mirr, 1, 1, 0, log_count,
-			    allocatable_pvs, alloc, MIRROR_BY_SEG)) {
+	if (!lv_add_mirrors(cmd, lv_mirr, NULL, 1u, 1u, 0u, 0u, log_count,
+			    allocatable_pvs, alloc, MIRROR_BY_SEG, 0)) {
 		log_error("Failed to convert pvmove LV to mirrored");
 		return_NULL;
 	}

Hi,

I'm interested in the mirrored log approach which Malahal posted, and now I'm looking into it. However, I found one problem with it.

When one of log disk is broken and is not recognized, there is a case disk that replication is executed. Let me explain with the following simple case, which is the mirror volume, vg00-lv00 is composed of two data disks and one mirrored log which is composed of two log disks.

vg00-lv00 (253:5)

    +vg00-lv00_mimage_1 (253:4)
    +vg00-lv00_mimage_0 (253:3)
    +vg00-lv00_mlog (253:2)
       +vg00-lv00_mlog_mimage_1 (253:1)
       +vg00-lv00_mlog_mimage_0 (253:0)

In this structure, if vg00-lv00_mlog_mimage_0 has problem and is not accessed successfully, vg00-lv00_mlog is detected as a failed device when vg00-lv00 is activated. The status of log device in vg00-lv00 is displayed as "D". In addition, a whole data in vg00-lv00_mimage0 is copied to vg00-lv00_mimage_1.

  # vgchange -ay vg00 -P
    Partial mode. Incomplete volume groups will be activated read-only.
    Couldn't find device with uuid 'aLl1CE-69hT-Mghu-UvqX-aBpA-twhk-jXO5Ut'.
    Couldn't find device with uuid 'aLl1CE-69hT-Mghu-UvqX-aBpA-twhk-jXO5Ut'.
    Couldn't find device with uuid 'aLl1CE-69hT-Mghu-UvqX-aBpA-twhk-jXO5Ut'.
    Couldn't find device with uuid 'aLl1CE-69hT-Mghu-UvqX-aBpA-twhk-jXO5Ut'.
    1 logical volume(s) in volume group "vg00" now active

  # dmsetup status
  vg00-lv00_mimage_1: 0 24576 linear 
  vg00-lv00_mlog_mimage_1: 0 8192 linear 
  vg00-lv00_mimage_0: 0 24576 linear 
  vg00-lv00_mlog_mimage_0: 0 8192 linear 
  vg00-lv00_mlog_mimage_0-missing_0_0: 0 8192 error 
  vg00-lv00_mlog: 0 8192 mirror 2 253:1 253:2 0/8 1 DA 1 core
  vg00-lv00: 0 24576 mirror 2 253:4 253:5 24/24 1 AA 3 disk 253:3 D

On the other hand, when vg00-lv00_mlog_mimage_0 is in a good condition but vg00-lv00_mlog_mimage_1 has a issue, vg00-lv00_mlog is recognized as an active device, and the status of log device in vg00-lv00 is displayed as "A". This is an expected result.

  # /sbin/dmsetup status
  vg00-lv00_mimage_1: 0 24576 linear 
  vg00-lv00_mlog_mimage_1: 0 8192 linear 
  vg00-lv00_mimage_0: 0 24576 linear 
  vg00-lv00_mlog_mimage_0: 0 8192 linear 
  vg00-lv00_mlog: 0 8192 mirror 2 253:0 253:2 0/8 1 AD 1 core
  vg00-lv00: 0 24576 mirror 2 253:4 253:5 24/24 1 AA 3 disk 253:3 A
  vg00-lv00_mlog_mimage_1-missing_0_0: 0 8192 error 

* Analysis of this problem

A mirrored log is a type of "core" log and log devices need to be synchronized when a mirrored log is activated. But when the first log device is not recognized, "READ" I/O returns -EIO in disk_resume() because log disk is not in-sync status and a default log can not be switched to the other log disk working well.

Avoiding disk replication even if a log device got trouble is one of the requirements. Is there any solution to avoid this problem by the mirrored log approach?

I appreciate any comments on this problem.

Thanks,
---
Takahiro Yasui
Hitachi Computer Products (America) Inc.

Two ways to fix:
1) Never make "error leg" as your master leg as that is pointless.
2) Maybe we can use 'nosync' option when one of the two legs is known to be an error device. This is probably easier than method (1)

Any other methods? Any help regarding which method to pursue would be very much appreciated.

Thanks, Malahal.
PS: This patch was always used along with other patches and that could explain why we didn't notice this problem.

Hi Malahal,

If I understand Takahiro correctly, the main issue is that the whole log device becomes unusable in case one leg is broken...? Of course, we still have the other leg with the most recent data available and can read from that one. However, this would require that we can start a mirror in rw-mode (so we can keep the still-recent half up to date) even if one half has failed - a functionality which was recently added to LVM, as we discussed in our last call...? I'm not sure if I understand his statements about the synchronization correctly. Why would any synchronization within the data mirror occur at all? It's only the log which is broken, not the actual data in vg00-lv00_mimage_0/1. If any synchronization in the _data_ mirror occurs just because one leg in the _log_ mirror has a problem, I'd consider that a bug.

Ciao,
Stefan

Linux on System z
IBM Deutschland Research & Development GmbH
Vorsitzender des Aufsichtsrats: Martin Jetter
Geschäftsführer: Erich Baier
Sitz der Gesellschaft: Böblingen
Registergericht: Amtsgericht Stuttgart, HRB 243294

Stefan, Takahiro is using the latest LVM that brings LVs in rw-mode. That is not the problem. Actually, everything works if a slave leg of a "log mirror" is broken. But if the master leg of the "log mirror" is broken, entire log device becomes unusable. The reason is when we activate the "log mirror", it always synchronizes the data from the master leg to slave leg since it has no log device (as the "log mirror" uses "corelog"). In other words, the "log mirror" always starts in "not in sync" state and synchronizes.

If the master leg is broken for the "log mirror", it starts in "not in sync" state and tries to copy data from the "broken leg" to the active leg! This appears as the master leg failure while doing synchronization which makes the whole device unusable.

Since the entire log device is unusable with its "master device broken", the "data mirror" goes for synchronization.

Thank you for the good explanation, Malahal. You are explaining exactly what I need to say. I'm using LVM2.2.02.43, on which this mirroredlog patch is built.

Thanks,
Taka

This approach will solve the case that a log leg is not recognized, but I'm afraid that it won't solve other type of I/O errors, such as medium error.

LVM commands such as vgchange accesses only sectors containing label and metadata, and LVM commands will successfully activate mirrored log as a userland process. However, in kernel, I/Os errors might be detected in a log area, and "READ" I/O returns -EIO. In this case, log will be detected as a failed device.

This approach looks good.

Thanks,
---
Takahiro Yasui
Hitachi Computer Products (America), Inc.

Hi, Malahal

This is the additional test results of mirrored log. Currently your patch is proposed by "RFC", and these might be known issues to you, but I hope this reports would be helpful to improve your patch. Any comments are welcome.

0. Test environment
- LVM2.2.02.43 with mirrored log patch
- vg00 is composed of 8 PVs (/dev/sd[c-j])

1. SIGSEGV at converting a core log to a mirrored log

  # dmsetup status
  vg00-lv00_mimage_1: 0 24576 linear
  vg00-lv00_mimage_0: 0 24576 linear
  vg00-lv00: 0 24576 mirror 2 253:3 253:4 24/24 1 AA 1 core

  # lvconvert -m1 --mirroredlog /dev/vg00/lv00
  Segmentation fault

  # lvconvert -m2 --mirroredlog /dev/vg00/lv00
  Segmentation fault

2. Fail at conversion from 2 legs to 3 legs

lv00 in vg00 has 2 legs (2 log legs and 2 data legs), and when it is converted to the structure with 3 lets, the conversion fails.

vg00-lv00 (253:5)
+ vg00-lv00_mimage_1 (253:4)
+ vg00-lv00_mimage_0 (253:3)
+ vg00-lv00_mlog (253:2)
+ vg00-lv00_mlog_mimage_1 (253:1)
+ vg00-lv00_mlog_mimage_0 (253:0)

  # lvconvert -m2 --mirroredlog /dev/vg00/lv00
    device-mapper: reload ioctl failed: No such device or address
    Failed to lock lv00
  # dmsetup status
  vg00-lv00_mimage_2: 0 8192 linear    <-- data leg is created
  vg00-lv00_mimage_1: 0 24576 linear
  vg00-lv00_mlog_mimage_1: 0 8192 linear
  vg00-lv00_mimage_0: 0 24576 linear
  vg00-lv00_mlog_mimage_0: 0 8192 linear
  vg00-lv00_mimagetmp_2: 0 24576 error <-- temporary data log exists
  vg00-lv00_mlog: 0 8192 mirror 2 253:1 253:2 8/8 1 AA 1 core
  vg00-lv00: 0 24576 mirror 2 253:4 253:5 24/24 1 AA 3 disk 253:3 A

3. Fail at conversion from disk log to mirrored log

  # dmsetup status
  vg00-lv00_mimage_1: 0 24576 linear
  vg00-lv00_mimage_0: 0 24576 linear
  vg00-lv00_mlog: 0 8192 linear
  vg00-lv00: 0 24576 mirror 2 253:1 253:2 24/24 1 AA 3 disk 253:0 A

  # lvconvert -m1 --mirroredlog /dev/vg00/lv00
  Logical volume lv00 already has 1 mirror(s).

=> disk log should be converted to mirrored log?

  # lvconvert -m2 --mirroredlog /dev/vg00/lv00
  device-mapper: reload ioctl failed: No such device or address
  Failed to lock lv00
  # dmsetup status
  vg00-lv00_mimage_2: 0 8192 linear
  vg00-lv00_mimage_1: 0 24576 linear
  vg00-lv00_mimage_0: 0 24576 linear
  vg00-lv00_mimagetmp_2: 0 24576 error
  vg00-lv00_mlog: 0 8192 linear
  vg00-lv00: 0 24576 mirror 2 253:1 253:2 24/24 1 AA 3 disk 253:0 A

=> disk log should be converted to mirrored log with 3 legs?

Thanks,
---
Takahiro Yasui
Hitachi Computer Products (America), Inc.

Thank you Takahiro for your comments. They all need to be fixed! I also found that 'lvremove' doesn't remove the device mapper tables for the "log mirror".

--Malahal.

Be careful not to waste your time on some thing that will not be included....

I see the order of events like this:
1) Generic layering of LVs
2) Improved mirror DSO
3) Mirrored log implementation

Note that once you get #1 done you /should/ be able to get a mirrored log by:
prompt> lvcreate -m1 --mirrorlog core -l 1 -n mirrored_log vg prompt> lvcreate -m1 -L 5G -n mirror vg /dev/sdb /dev/sdc /dev/vg/mirrored_log

Leapfrogging #2 /might/ be acceptable, but leapfrogging #1 is not. This is because you are creating a one-off solution rather than creating something more general that would benefit LVM as a whole.

Feel free to disagree, but I really see the above as the way forward.

brassow

Jonathan, what exactly is "Generic layering of LVs"? What is there already and what is not there. Can you please explain with details.

Thanks, Malahal.