Skip to contentSkip to conversation

Share vg metadata for read only

PATCH New Zdenek Kabelac 3 participants 7 messages Feb 23, 2011

lore
Share & tools
lore
#0 lore raw

cover Share vg metadata for read only

Another nice speedup:

1st. patch refactors allocation of VG to one place - so refcounting is simplier to implement (I'm planning few more cleanup patches for this)

2nd. patch adds reference counting and caching of VG struct
As Milan suggested - I'll try to add extra code which will try to check checksum of memory pool used for VG (with DEBUG_MEM option probably)

Some numbers with ~1700 volumes and current default rawhide kernel:
2.6.38-0.rc5.git1.1.fc15.x86_64 lvm2-2.02.84-1.fc15.x86_64
(killed gvfs... and removed udisk udev rules
- as these are currently major performance killers)

vgchange -ay
real 0m50.097s
user 0m10.765s
sys 0m35.481s

vgchange -ay (here are LVs already actived)
real 0m10.425s
user 0m8.240s
sys 0m2.133s

vgchange -an
real 0m51.339s
user 0m10.375s
sys 0m31.457s

vgchange -an (here are LVs already deactived)
real 0m10.288s
user 0m8.328s
sys 0m1.901s

and now the same with new lvm2 sharing code (and critical section)

vgchange -ay
real 0m13.792s (udev is slowing it down by 3 seconds)
user 0m0.629s
sys 0m4.535s

vgchange -ay
real 0m0.745s
user 0m0.170s
sys 0m0.520s

vgchange -an
real 0m9.886s
user 0m0.425s
sys 0m2.418s

vgchange -an
real 0m0.468s
user 0m0.158s
sys 0m0.239s

Zdenek Kabelac (3):
Refactor vg allocation code
Add VG refcount and cache created VG structure
Destroy 'fic' memory in error path

 lib/cache/lvmcache.c          |   55 +++++++++++++++++++++++++++-------------
 lib/cache/lvmcache.h          |    4 ++-
 lib/format_text/import_vsn1.c |   33 +++++++-----------------
 lib/metadata/metadata.c       |   40 ++++++-----------------------
 lib/metadata/vg.c             |   26 +++++++++++++++++++
 lib/metadata/vg.h             |    3 ++
 6 files changed, 88 insertions(+), 73 deletions(-)

--
1.7.4.1

#1 lore raw

PATCH 1/3 Refactor vg allocation code

Create new function to allocate VG structure.

Usure about naming strategy - we use i.e. alloc_lv, but most the new function have now operation prefix.

Also the location after Dave's refactoring patches is unclear.

I'd suppose we should move all 'vg' related code to vg.c, but for now lot's of stuff is still spread across metadata.c and other files (i.e. free_vg).

---
 lib/format_text/import_vsn1.c |   33 ++++++++++-----------------------
 lib/metadata/metadata.c       |   35 +++++------------------------------
 lib/metadata/vg.c             |   25 +++++++++++++++++++++++++
 lib/metadata/vg.h             |    2 ++
 4 files changed, 42 insertions(+), 53 deletions(-)
lib/format_text/import_vsn1.c +10 -23
index 0ffe334..bc63f36 100644
@@ -652,35 +652,28 @@ static struct volume_group *_read_vg(struct format_instance *fid,
 	const struct config_node *vgn, *cn;
 	struct volume_group *vg;
 	struct dm_hash_table *pv_hash = NULL, *lv_hash = NULL;
-	struct dm_pool *mem = dm_pool_create("lvm2 vg_read", VG_MEMPOOL_CHUNK);
 	unsigned scan_done_once = use_cached_pvs;
 
-	if (!mem)
-		return_NULL;
-
 	/* skip any top-level values */
 	for (vgn = cft->root; (vgn && vgn->v); vgn = vgn->sib)
 		;
 
 	if (!vgn) {
 		log_error("Couldn't find volume group in file.");
-		goto bad;
+		return NULL;
 	}
 
-	if (!(vg = dm_pool_zalloc(mem, sizeof(*vg))))
-		goto_bad;
-
-	vg->vgmem = mem;
-	vg->cmd = fid->fmt->cmd;
+	if (!(vg = alloc_vg(fid->fmt->cmd)))
+		return_NULL;
 
 	/* FIXME Determine format type from file contents */
 	/* eg Set to instance of fmt1 here if reading a format1 backup? */
 	vg->fid = fid;
 
-	if (!(vg->name = dm_pool_strdup(mem, vgn->key)))
+	if (!(vg->name = dm_pool_strdup(vg->vgmem, vgn->key)))
 		goto_bad;
 
-	if (!(vg->system_id = dm_pool_zalloc(mem, NAME_LEN)))
+	if (!(vg->system_id = dm_pool_zalloc(vg->vgmem, NAME_LEN)))
 		goto_bad;
 
 	vgn = vgn->child;
@@ -733,7 +726,6 @@ static struct volume_group *_read_vg(struct format_instance *fid,
 		goto bad;
 	}
 
-	vg->alloc = ALLOC_NORMAL;
 	if ((cn = find_config_node(vgn, "allocation_policy"))) {
 		const struct config_value *cv = cn->v;
 		if (!cv || !cv->v.str) {
@@ -761,21 +753,16 @@ static struct volume_group *_read_vg(struct format_instance *fid,
 		goto bad;
 	}
 
-	dm_list_init(&vg->pvs);
-	if (!_read_sections(fid, "physical_volumes", _read_pv, mem, vg,
+	if (!_read_sections(fid, "physical_volumes", _read_pv, vg->vgmem, vg,
 			    vgn, pv_hash, lv_hash, 0, &scan_done_once)) {
 		log_error("Couldn't find all physical volumes for volume "
 			  "group %s.", vg->name);
 		goto bad;
 	}
 
-	dm_list_init(&vg->lvs);
-	dm_list_init(&vg->tags);
-	dm_list_init(&vg->removed_pvs);
-
 	/* Optional tags */
 	if ((cn = find_config_node(vgn, "tags")) &&
-	    !(read_tags(mem, &vg->tags, cn->v))) {
+	    !(read_tags(vg->vgmem, &vg->tags, cn->v))) {
 		log_error("Couldn't read tags for volume group %s.", vg->name);
 		goto bad;
 	}
@@ -789,14 +776,14 @@ static struct volume_group *_read_vg(struct format_instance *fid,
 		goto bad;
 	}
 
-	if (!_read_sections(fid, "logical_volumes", _read_lvnames, mem, vg,
+	if (!_read_sections(fid, "logical_volumes", _read_lvnames, vg->vgmem, vg,
 			    vgn, pv_hash, lv_hash, 1, NULL)) {
 		log_error("Couldn't read all logical volume names for volume "
 			  "group %s.", vg->name);
 		goto bad;
 	}
 
-	if (!_read_sections(fid, "logical_volumes", _read_lvsegs, mem, vg,
+	if (!_read_sections(fid, "logical_volumes", _read_lvsegs, vg->vgmem, vg,
 			    vgn, pv_hash, lv_hash, 1, NULL)) {
 		log_error("Couldn't read all logical volumes for "
 			  "volume group %s.", vg->name);
@@ -824,7 +811,7 @@ static struct volume_group *_read_vg(struct format_instance *fid,
 	if (lv_hash)
 		dm_hash_destroy(lv_hash);
 
-	dm_pool_destroy(mem);
+	free_vg(vg);
 	return NULL;
 }
 
lib/metadata/metadata.c +5 -30
index 6a29d24..d75a1cf 100644
@@ -942,18 +942,8 @@ static struct volume_group *_vg_make_handle(struct cmd_context *cmd,
 			     struct volume_group *vg,
 			     uint32_t failure)
 {
-	struct dm_pool *vgmem;
-
-	if (!vg) {
-		if (!(vgmem = dm_pool_create("lvm2 vg_handle", VG_MEMPOOL_CHUNK)) ||
-		    !(vg = dm_pool_zalloc(vgmem, sizeof(*vg)))) {
-			log_error("Error allocating vg handle.");
-			if (vgmem)
-				dm_pool_destroy(vgmem);
-			return_NULL;
-		}
-		vg->vgmem = vgmem;
-	}
+	if (!vg && !(vg = alloc_vg(cmd)))
+		return_NULL;
 
 	vg->read_status = failure;
 
@@ -994,7 +984,6 @@ struct volume_group *vg_create(struct cmd_context *cmd, const char *vg_name)
 	struct volume_group *vg;
 	struct format_instance_ctx fic;
 	int consistent = 0;
-	struct dm_pool *mem;
 	uint32_t rc;
 
 	if (!validate_name(vg_name)) {
@@ -1017,10 +1006,7 @@ struct volume_group *vg_create(struct cmd_context *cmd, const char *vg_name)
 		return _vg_make_handle(cmd, NULL, FAILED_EXIST);
 	}
 
-	if (!(mem = dm_pool_create("lvm2 vg_create", VG_MEMPOOL_CHUNK)))
-		goto_bad;
-
-	if (!(vg = dm_pool_zalloc(mem, sizeof(*vg))))
+        if (!(vg = alloc_vg(cmd)))
 		goto_bad;
 
 	if (!id_create(&vg->id)) {
@@ -1032,16 +1018,13 @@ struct volume_group *vg_create(struct cmd_context *cmd, const char *vg_name)
 	/* Strip dev_dir if present */
 	vg_name = strip_dir(vg_name, cmd->dev_dir);
 
-	vg->vgmem = mem;
-	vg->cmd = cmd;
-
-	if (!(vg->name = dm_pool_strdup(mem, vg_name)))
+	if (!(vg->name = dm_pool_strdup(vg->vgmem, vg_name)))
 		goto_bad;
 
 	vg->seqno = 0;
 
 	vg->status = (RESIZEABLE_VG | LVM_READ | LVM_WRITE);
-	if (!(vg->system_id = dm_pool_alloc(mem, NAME_LEN)))
+	if (!(vg->system_id = dm_pool_alloc(vg->vgmem, NAME_LEN)))
 		goto_bad;
 
 	*vg->system_id = '\0';
@@ -1057,14 +1040,6 @@ struct volume_group *vg_create(struct cmd_context *cmd, const char *vg_name)
 	vg->mda_copies = DEFAULT_VGMETADATACOPIES;
 
 	vg->pv_count = 0;
-	dm_list_init(&vg->pvs);
-
-	dm_list_init(&vg->lvs);
-
-	dm_list_init(&vg->tags);
-
-	/* initialize removed_pvs list */
-	dm_list_init(&vg->removed_pvs);
 
 	fic.type = FMT_INSTANCE_VG | FMT_INSTANCE_MDAS | FMT_INSTANCE_AUX_MDAS;
 	fic.context.vg_ref.vg_name = vg_name;
lib/metadata/vg.c +25 -0
index 9d0d5dd..392d23c 100644
@@ -18,6 +18,31 @@
 #include "display.h"
 #include "activate.h"
 
+struct volume_group *alloc_vg(struct cmd_context *cmd)
+{
+	struct dm_pool *vgmem;
+	struct volume_group *vg;
+
+	if (!(vgmem = dm_pool_create("lvm2 vg_handle", VG_MEMPOOL_CHUNK)) ||
+	    !(vg = dm_pool_zalloc(vgmem, sizeof(*vg)))) {
+		log_error("Error allocating VG structure.");
+		if (vgmem)
+			dm_pool_destroy(vgmem);
+		return NULL;
+	}
+
+	vg->cmd = cmd;
+	vg->vgmem = vgmem;
+	vg->alloc = ALLOC_NORMAL;
+
+	dm_list_init(&vg->pvs);
+	dm_list_init(&vg->lvs);
+	dm_list_init(&vg->tags);
+	dm_list_init(&vg->removed_pvs);
+
+	return vg;
+}
+
 char *vg_fmt_dup(const struct volume_group *vg)
 {
 	if (!vg->fid || !vg->fid->fmt)
lib/metadata/vg.h +2 -0
index 448dcfe..c00fb74 100644
@@ -95,6 +95,8 @@ struct volume_group {
 	uint32_t mda_copies; /* target number of mdas for this VG */
 };
 
+struct volume_group *alloc_vg(struct cmd_context *cmd);
+
 char *vg_fmt_dup(const struct volume_group *vg);
 char *vg_name_dup(const struct volume_group *vg);
 char *vg_system_id_dup(const struct volume_group *vg);
Signature · 1 line
1.7.4.1
#4 lore raw

PATCH 2/3 Add VG refcount and cache created VG structure

In reply to Zdenek Kabelac

Imported VG from config tree is now cached in lvmcache_vginfo for read-only code path..

Extended lvmcache_get_vg() with new parameter 'consistent'. It passing information about how the metadata are going to be used. For 'read-only' case it is == '0'.

VG structure has reference counter to support shared usage of VG. (One copy is kept by cache - and 2nd is processed by the code and released with free_vg()).

Sharing of 'write' MDA is not being made - after writting, metadata cache is flushed and new data is read back and parsed again.

---
 lib/cache/lvmcache.c    |   53 +++++++++++++++++++++++++++++++----------------
 lib/cache/lvmcache.h    |    4 ++-
 lib/metadata/metadata.c |    5 +++-
 lib/metadata/vg.c       |    1 +
 lib/metadata/vg.h       |    1 +
 5 files changed, 44 insertions(+), 20 deletions(-)
lib/cache/lvmcache.c +35 -18
index 0a8693c..7c53b14 100644
@@ -89,6 +89,11 @@ static void _free_cached_vgmetadata(struct lvmcache_vginfo *vginfo)
 		vginfo->cft = NULL;
 	}
 
+	if (vginfo->vg) {
+		free_vg(vginfo->vg);
+		vginfo->vg = NULL;
+	}
+
 	log_debug("Metadata cache: VG %s wiped.", vginfo->vgname);
 }
 
@@ -624,7 +629,7 @@ int lvmcache_label_scan(struct cmd_context *cmd, int full_scan)
 	return r;
 }
 
-struct volume_group *lvmcache_get_vg(const char *vgid, unsigned precommitted)
+struct volume_group *lvmcache_get_vg(const char *vgid, int consistent, unsigned precommitted)
 {
 	struct lvmcache_vginfo *vginfo;
 	struct volume_group *vg;
@@ -653,25 +658,37 @@ struct volume_group *lvmcache_get_vg(const char *vgid, unsigned precommitted)
 	    (!precommitted && vginfo->precommitted && !critical_section()))
 		return NULL;
 
-	fic.type = FMT_INSTANCE_VG | FMT_INSTANCE_MDAS | FMT_INSTANCE_AUX_MDAS;
-	fic.context.vg_ref.vg_name = vginfo->vgname;
-	fic.context.vg_ref.vg_id = vgid;
-	if (!(fid = vginfo->fmt->ops->create_instance(vginfo->fmt, &fic)))
-		return_NULL;
+	if (vginfo->vg && (consistent == 0)) {
+		vg = vginfo->vg;
+		++vg->refcount;  /* Share with caller */
+		log_debug("Sharing VG %s (%d)", vginfo->vgname, vg->refcount);
+	} else {
+		fic.type = FMT_INSTANCE_VG | FMT_INSTANCE_MDAS | FMT_INSTANCE_AUX_MDAS;
+		fic.context.vg_ref.vg_name = vginfo->vgname;
+		fic.context.vg_ref.vg_id = vgid;
+		if (!(fid = vginfo->fmt->ops->create_instance(vginfo->fmt, &fic)))
+			return_NULL;
 
-	/* Build config tree from vgmetadata, if not yet cached */
-	if (!vginfo->cft &&
-	    !(vginfo->cft =
-	      create_config_tree_from_string(fid->fmt->cmd,
-					     vginfo->vgmetadata))) {
-		_free_cached_vgmetadata(vginfo);
-		return_NULL;
-	}
+		/* Build config tree from vgmetadata, if not yet cached */
+		if (!vginfo->cft &&
+		    !(vginfo->cft =
+		      create_config_tree_from_string(fid->fmt->cmd,
+						     vginfo->vgmetadata))) {
+			_free_cached_vgmetadata(vginfo);
+			return_NULL;
+		}
 
-	if (!(vg = import_vg_from_config_tree(vginfo->cft, fid))) {
-		_free_cached_vgmetadata(vginfo);
-		return_NULL;
-	}
+		if (!(vg = import_vg_from_config_tree(vginfo->cft, fid))) {
+			_free_cached_vgmetadata(vginfo);
+			return_NULL;
+		}
+
+		/* Cache only 'read-only' imported VG metadata */
+		if (consistent == 0) {
+			vginfo->vg = vg;
+			++vg->refcount; /* Keep copy for cache */
+		}
+	} 
 
 	log_debug("Using cached %smetadata for VG %s.",
 		  vginfo->precommitted ? "pre-committed" : "", vginfo->vgname);
lib/cache/lvmcache.h +3 -1
index 080f3b5..19ef423 100644
@@ -49,6 +49,8 @@ struct lvmcache_vginfo {
 	char *vgmetadata;	/* Copy of VG metadata as format_text string */
 	struct config_tree *cft; /* Config tree created from vgmetadata */
 				/* Lifetime is directly tied to vgmetadata */
+	struct volume_group *vg; /* Cached parsed VG structure */
+				/* Lifetime is directly tied to vgmetadata */
 	unsigned precommitted;	/* Is vgmetadata live or precommitted? */
 };
 
@@ -120,7 +122,7 @@ struct dm_list *lvmcache_get_pvids(struct cmd_context *cmd, const char *vgname,
 				const char *vgid);
 
 /* Returns cached volume group metadata. */
-struct volume_group *lvmcache_get_vg(const char *vgid, unsigned precommitted);
+struct volume_group *lvmcache_get_vg(const char *vgid, int consistent, unsigned precommitted);
 void lvmcache_drop_metadata(const char *vgname, int drop_precommitted);
 void lvmcache_commit_metadata(const char *vgname);
 
lib/metadata/metadata.c +4 -1
index d75a1cf..563f0cb 100644
@@ -2860,7 +2860,7 @@ static struct volume_group *_vg_read(struct cmd_context *cmd,
 	 * Also return if use_precommitted is set due to the FIXME in
 	 * the missing PV logic below.
 	 */
-	if ((correct_vg = lvmcache_get_vg(vgid, precommitted)) &&
+	if ((correct_vg = lvmcache_get_vg(vgid, *consistent, precommitted)) &&
 	    (use_precommitted || !*consistent || !(correct_vg->status & INCONSISTENT_VG))) {
 		if (!(correct_vg->status & INCONSISTENT_VG))
 			*consistent = 1;
@@ -3272,6 +3272,9 @@ void free_vg(struct volume_group *vg)
 	if (!vg)
 		return;
 
+	if (--vg->refcount)
+		return;
+
 	if (vg->cmd && vg->vgmem == vg->cmd->mem) {
 		log_error(INTERNAL_ERROR "global memory pool used for VG %s",
 			  vg->name);
lib/metadata/vg.c +1 -0
index 392d23c..e21645c 100644
@@ -31,6 +31,7 @@ struct volume_group *alloc_vg(struct cmd_context *cmd)
 		return NULL;
 	}
 
+	vg->refcount = 1;
 	vg->cmd = cmd;
 	vg->vgmem = vgmem;
 	vg->alloc = ALLOC_NORMAL;
lib/metadata/vg.h +1 -0
index c00fb74..ad6e071 100644
@@ -37,6 +37,7 @@ struct volume_group {
 	struct format_instance *fid;
 	struct dm_list *cmd_vgs;/* List of wanted/locked and opened VGs */
 	uint32_t cmd_missing_vgs;/* Flag marks missing VG */
+	uint32_t refcount;	/* Reference counter */
 	uint32_t seqno;		/* Metadata sequence number */
 
 	alloc_policy_t alloc;
Signature · 1 line
1.7.4.1
#5 lore raw

PATCH 3/3 Destroy 'fic' memory in error path

In reply to Zdenek Kabelac

As now FIC allocates hash table - we need to properly destroy fid on error path.

---
 lib/cache/lvmcache.c |    2 ++
 1 files changed, 2 insertions(+), 0 deletions(-)
lib/cache/lvmcache.c +2 -0
index 7c53b14..3ce39f9 100644
@@ -674,11 +674,13 @@ struct volume_group *lvmcache_get_vg(const char *vgid, int consistent, unsigned
 		    !(vginfo->cft =
 		      create_config_tree_from_string(fid->fmt->cmd,
 						     vginfo->vgmetadata))) {
+			fid->fmt->ops->destroy_instance(fid);
 			_free_cached_vgmetadata(vginfo);
 			return_NULL;
 		}
 
 		if (!(vg = import_vg_from_config_tree(vginfo->cft, fid))) {
+			fid->fmt->ops->destroy_instance(fid);
 			_free_cached_vgmetadata(vginfo);
 			return_NULL;
 		}
Signature · 1 line
1.7.4.1
#6 lore raw

PATCH 3/3 Destroy 'fic' memory in error path

Review

Quoted 1 line · Zdenek Kabelac
(quoted diff) +1 -0
+			fid->fmt->ops->destroy_instance(fid);

...and I'll send a patch that adds the content to destroy_instance since it's pure void now :)

Also, we need to add destroy_instance everywhere where necessary throughout the code. We haven't used this call before because we relied on automatic mempool deallocation mostly.

Peter

#2 lore raw

PATCH 1/3 Refactor vg allocation code

In reply to Zdenek Kabelac
Quoted 2 lines · Zdenek Kabelac

Usure about naming strategy - we use i.e. alloc_lv, but most the new function have now operation prefix.

alloc_vg for now...because it's paired with free_vg and vg_free is already used for something else:)

Quoted 1 line

Also the location after Dave's refactoring patches is unclear.

vg.c - move free_vg there too.

lib/metadata/vg.c line 26

Quoted 1 line
lib/metadata/vg.c +1 -0
+	if (!(vgmem = dm_pool_create("lvm2 vg_handle", VG_MEMPOOL_CHUNK)) ||

The code is now shared - "alloc_vg" will have to do, or if that makes the log too hard to understand, pass the existing strings into the function.

Ack.

Alasdair

#3 lore raw

PATCH 1/3 Refactor vg allocation code

Dne 23.2.2011 17:48, Alasdair G Kergon napsal(a):

Quoted 7 lines · Zdenek Kabelac
Quoted 2 lines

Usure about naming strategy - we use i.e. alloc_lv, but most the new function have now operation prefix.

alloc_vg for now...because it's paired with free_vg and vg_free is already used for something else:)

Quoted 8 lines
Quoted 1 line

Also the location after Dave's refactoring patches is unclear.

vg.c - move free_vg there too.

lib/metadata/vg.c line 26

Quoted 1 line
lib/metadata/vg.c +1 -0
+	if (!(vgmem = dm_pool_create("lvm2 vg_handle", VG_MEMPOOL_CHUNK)) ||

The code is now shared - "alloc_vg" will have to do, or if that makes the log too hard to understand, pass the existing strings into the function.

Ok - I'll add string arg to alloc_vg() for debug purposes.

Zdenek