Skip to contentSkip to conversation

Properly decode flags even for VG locks.

PATCH New Milan Broz 1 participant 4 messages Dec 28, 2009

lore
Share & tools
lore

Milan Broz

author
#0 lore raw

PATCH 1/4 Properly decode flags even for VG locks.

And decode flags in humar readable form in client. And clean some trailing whitespaces.

No eal change n this patch (only debugging messages changed).

---
 daemons/clvmd/lvm-functions.c |   71 +++++++++++++++++++++--------------------
 lib/locking/cluster_locking.c |   13 ++++---
 2 files changed, 43 insertions(+), 41 deletions(-)
daemons/clvmd/lvm-functions.c +36 -35
index 385aa86..c832ec5 100644
@@ -74,23 +74,23 @@ static const char *decode_locking_cmd(unsigned char cmdl)
 	const char *command;
 
 	switch (cmdl & LCK_TYPE_MASK) {
-	case LCK_NULL:   
-		type = "NULL";   
+	case LCK_NULL:
+		type = "NULL";
 		break;
-	case LCK_READ:   
-		type = "READ";   
+	case LCK_READ:
+		type = "READ";
 		break;
-	case LCK_PREAD:  
-		type = "PREAD";  
+	case LCK_PREAD:
+		type = "PREAD";
 		break;
-	case LCK_WRITE:  
-		type = "WRITE";  
+	case LCK_WRITE:
+		type = "WRITE";
 		break;
-	case LCK_EXCL:   
-		type = "EXCL";   
+	case LCK_EXCL:
+		type = "EXCL";
 		break;
-	case LCK_UNLOCK: 
-		type = "UNLOCK"; 
+	case LCK_UNLOCK:
+		type = "UNLOCK";
 		break;
 	default:
 		type = "unknown";
@@ -98,34 +98,35 @@ static const char *decode_locking_cmd(unsigned char cmdl)
 	}
 
 	switch (cmdl & LCK_SCOPE_MASK) {
-	case LCK_VG: 
-		scope = "VG"; 
+	case LCK_VG:
+		scope = "VG";
+		command = "LCK_VG";
 		break;
-	case LCK_LV: 
+	case LCK_LV:
 		scope = "LV"; 
+		switch (cmdl & LCK_MASK) {
+		case LCK_LV_EXCLUSIVE & LCK_MASK:
+			command = "LCK_LV_EXCLUSIVE";
+			break;
+		case LCK_LV_SUSPEND & LCK_MASK:
+			command = "LCK_LV_SUSPEND";
+			break;
+		case LCK_LV_RESUME & LCK_MASK:
+			command = "LCK_LV_RESUME";
+			break;
+		case LCK_LV_ACTIVATE & LCK_MASK:
+			command = "LCK_LV_ACTIVATE";
+			break;
+		case LCK_LV_DEACTIVATE & LCK_MASK:
+			command = "LCK_LV_DEACTIVATE";
+			break;
+		default:
+			command = "unknown";
+			break;
+		}
 		break;
 	default:
 		scope = "unknown";
-		break;
-	}
-
-	switch (cmdl & LCK_MASK) {
-	case LCK_LV_EXCLUSIVE & LCK_MASK:
-		command = "LCK_LV_EXCLUSIVE";  
-		break;
-	case LCK_LV_SUSPEND & LCK_MASK:    
-		command = "LCK_LV_SUSPEND";    
-		break;
-	case LCK_LV_RESUME & LCK_MASK:     
-		command = "LCK_LV_RESUME";     
-		break;
-	case LCK_LV_ACTIVATE & LCK_MASK:   
-		command = "LCK_LV_ACTIVATE";   
-		break;
-	case LCK_LV_DEACTIVATE & LCK_MASK: 
-		command = "LCK_LV_DEACTIVATE"; 
-		break;
-	default:
 		command = "unknown";
 		break;
 	}
lib/locking/cluster_locking.c +7 -6
index d3138f2..225795a 100644
@@ -450,12 +450,13 @@ int lock_resource(struct cmd_context *cmd, const char *resource, uint32_t flags)
 		return 0;
 	}
 
-	log_very_verbose("Locking %s %s %s %s%s%s%s (0x%x)", lock_scope, lockname,
-			 lock_type,
-			 flags & LCK_NONBLOCK ? "" : "B",
-			 flags & LCK_HOLD ? "H" : "",
-			 flags & LCK_LOCAL ? "L" : "",
-			 flags & LCK_CLUSTER_VG ? "C" : "",
+	log_very_verbose("Locking %s %s %s (%s%s%s%s%s%s) (0x%x)", lock_scope, lockname,
+			 lock_type, lock_scope,
+			 flags & LCK_NONBLOCK ? "|NONBLOCK" : "",
+			 flags & LCK_HOLD ? "|HOLD" : "",
+			 flags & LCK_LOCAL ? "|LOCAL" : "",
+			 flags & LCK_CLUSTER_VG ? "|CLUSTER" : "",
+			 flags & LCK_CACHE ? "|CACHE" : "",
 			 flags);
 
 	/* Send a message to the cluster manager */
Signature · 1 line
1.6.5.7
author
#1 lore raw

PATCH 2/4 Move processing of VG locks to separate function (similar to LV locks).

And print some debugging info.

No functional change in this patch.

---
 daemons/clvmd/clvmd-command.c |    9 +++------
 daemons/clvmd/lvm-functions.c |   17 ++++++++++++++---
 daemons/clvmd/lvm-functions.h |    3 ++-
 3 files changed, 19 insertions(+), 10 deletions(-)
daemons/clvmd/clvmd-command.c +3 -6
index 2612a0e..1f8d0cf 100644
@@ -119,15 +119,12 @@ int do_command(struct local_client *client, struct clvm_header *msg, int msglen,
 		break;
 
 	case CLVMD_CMD_LOCK_VG:
+		lock_cmd = args[0];
+		lock_flags = args[1];
 		lockname = &args[2];
 		/* Check to see if the VG is in use by LVM1 */
 		status = do_check_lvm1(lockname);
-		/* P_#global causes a full cache refresh */
-		if (!strcmp(lockname, "P_" VG_GLOBAL))
-			do_refresh_cache();
-		else
-			drop_metadata(lockname + 2);
-
+		do_lock_vg(lock_cmd, lock_flags, lockname);
 		break;
 
 	case CLVMD_CMD_LOCK_LV:
daemons/clvmd/lvm-functions.c +14 -3
index c832ec5..983f90f 100644
@@ -676,12 +676,23 @@ static void drop_vg_locks()
 }
 
 /*
- * Drop lvmcache metadata
+ * Handle VG lock - drop metadata or update lvmcache state
  */
-void drop_metadata(const char *vgname)
+void do_lock_vg(unsigned char command, unsigned char lock_flags, char *resource)
 {
-	DEBUGLOG("Dropping metadata for VG %s\n", vgname);
+	char *vgname = resource + 2;
+
+	DEBUGLOG("do_lock_vg: resource '%s', cmd = %s, flags = %s, memlock = %d\n",
+		 resource, decode_locking_cmd(command), decode_flags(lock_flags), memlock());
+
+	/* P_#global causes a full cache refresh */
+	if (!strcmp(resource, "P_" VG_GLOBAL)) {
+		do_refresh_cache();
+		return;
+	}
+
 	pthread_mutex_lock(&lvm_lock);
+	DEBUGLOG("Dropping metadata for VG %s\n", vgname);
 	lvmcache_drop_metadata(vgname);
 	pthread_mutex_unlock(&lvm_lock);
 }
daemons/clvmd/lvm-functions.h +2 -1
index 3fc3e06..8b94e69 100644
@@ -35,6 +35,7 @@ extern void lvm_do_backup(const char *vgname);
 extern int hold_unlock(char *resource);
 extern int hold_lock(char *resource, int mode, int flags);
 extern char *get_last_lvm_error(void);
-extern void drop_metadata(const char *vgname);
+extern void do_lock_vg(unsigned char command, unsigned char lock_flags,
+		      char *resource);
 
 #endif
Signature · 1 line
1.6.5.7
author
#2 lore raw

PATCH 3/4 Add possibility to handle precommitted metadata in lvmcache.

In reply to Milan Broz

- Add drop_precommitted flag to force drop precommited metadata
- add lvmcache_commit_metadata() which upgrades precommitted metadata in cache

No functional change in this patch - just preparation for following change.

---
 daemons/clvmd/lvm-functions.c |    2 +-
 lib/cache/lvmcache.c          |   27 ++++++++++++++++++++-------
 lib/cache/lvmcache.h          |    3 ++-
 lib/locking/file_locking.c    |    2 +-
 4 files changed, 24 insertions(+), 10 deletions(-)
daemons/clvmd/lvm-functions.c +1 -1
index 983f90f..ad6b924 100644
@@ -693,7 +693,7 @@ void do_lock_vg(unsigned char command, unsigned char lock_flags, char *resource)
 
 	pthread_mutex_lock(&lvm_lock);
 	DEBUGLOG("Dropping metadata for VG %s\n", vgname);
-	lvmcache_drop_metadata(vgname);
+	lvmcache_drop_metadata(vgname, 0);
 	pthread_mutex_unlock(&lvm_lock);
 }
 
lib/cache/lvmcache.c +20 -7
index 358cd80..33e54ab 100644
@@ -158,7 +158,7 @@ static void _update_cache_lock_state(const char *vgname, int locked)
 	_update_cache_vginfo_lock_state(vginfo, locked);
 }
 
-static void _drop_metadata(const char *vgname)
+static void _drop_metadata(const char *vgname, int drop_precommitted)
 {
 	struct lvmcache_vginfo *vginfo;
 	struct lvmcache_info *info;
@@ -173,25 +173,38 @@ static void _drop_metadata(const char *vgname)
 	 * and we must not do it again.
 	 */
 
-	if (!vginfo->precommitted)
+	if (drop_precommitted || !vginfo->precommitted)
 		dm_list_iterate_items(info, &vginfo->infos)
 			info->status |= CACHE_INVALID;
 
 	_free_cached_vgmetadata(vginfo);
 }
 
-void lvmcache_drop_metadata(const char *vgname)
+/*
+ * Remote node uses this camm to upgrade precommited metadata to commited state
+ */
+void lvmcache_commit_metadata(const char *vgname)
+{
+	struct lvmcache_vginfo *vginfo;
+
+	if (!(vginfo = vginfo_from_vgname(vgname, NULL)))
+		return;
+
+	vginfo->precommitted = 0;
+}
+
+void lvmcache_drop_metadata(const char *vgname, int drop_precommitted)
 {
 	/* For VG_ORPHANS, we need to invalidate all labels on orphan PVs. */
 	if (!strcmp(vgname, VG_ORPHANS)) {
-		_drop_metadata(FMT_TEXT_ORPHAN_VG_NAME);
-		_drop_metadata(FMT_LVM1_ORPHAN_VG_NAME);
-		_drop_metadata(FMT_POOL_ORPHAN_VG_NAME);
+		_drop_metadata(FMT_TEXT_ORPHAN_VG_NAME, 0);
+		_drop_metadata(FMT_LVM1_ORPHAN_VG_NAME, 0);
+		_drop_metadata(FMT_POOL_ORPHAN_VG_NAME, 0);
 
 		/* Indicate that PVs could now be missing from the cache */
 		init_full_scan_done(0);
 	} else if (!vgname_is_locked(VG_GLOBAL))
-		_drop_metadata(vgname);
+		_drop_metadata(vgname, drop_precommitted);
 }
 
 /*
lib/cache/lvmcache.h +2 -1
index f1f1065..e2d749e 100644
@@ -111,6 +111,7 @@ struct dm_list *lvmcache_get_pvids(struct cmd_context *cmd, const char *vgname,
 
 /* Returns cached volume group metadata. */
 struct volume_group *lvmcache_get_vg(const char *vgid, unsigned precommitted);
-void lvmcache_drop_metadata(const char *vgname);
+void lvmcache_drop_metadata(const char *vgname, int drop_precommitted);
+void lvmcache_commit_metadata(const char *vgname);
 
 #endif
lib/locking/file_locking.c +1 -1
index 0ee8bbf..f47961d 100644
@@ -259,7 +259,7 @@ static int _file_lock_resource(struct cmd_context *cmd, const char *resource,
 	case LCK_VG:
 		/* Skip cache refresh for VG_GLOBAL - the caller handles it */
 		if (strcmp(resource, VG_GLOBAL))
-			lvmcache_drop_metadata(resource);
+			lvmcache_drop_metadata(resource, 0);
 
 		/* LCK_CACHE does not require a real lock */
 		if (flags & LCK_CACHE)
Signature · 1 line
1.6.5.7
author
#3 lore raw

PATCH 4/4 Propagate commit and revert metadata event to other nodes in cluster.

In reply to Milan Broz

This patch tries to corectly track changes in lvmcache related to commit/revert.

For vg_commit: if there are cached precommitted metadata, after successfull commit these metadata must be tracked as committed.

For vg_revert: remote nodes must drop precommitted metadata and its flag in lvmcache.

Because of current clvmd protocol limitation (unsigned char) there are some non-obvious hacks used.

- The LCK_CACHE bit is translated to another "optional" flag (because we are out of lock flags)
- The LOCK_HOLD (which is normally unused in VG locks) is used in commit/revert message

Also patch removes LCK_TYPES_MASKing which currently prevents these flas to be send to remote nodes.

(N.B. Patch do not touch LV locks here in any way.)

All this machinery is needed to properly solve remote node cache invalidaton which cause several problems recently observed.

---
 daemons/clvmd/lvm-functions.c |   25 ++++++++++++++++++++++---
 lib/locking/cluster_locking.c |    4 +++-
 lib/locking/locking.h         |   12 ++++++++++++
 lib/metadata/metadata.c       |    8 ++++++++
 4 files changed, 45 insertions(+), 4 deletions(-)
daemons/clvmd/lvm-functions.c +22 -3
index ad6b924..d7a3a18 100644
@@ -145,10 +145,11 @@ static const char *decode_flags(unsigned char flags)
 {
 	static char buf[128];
 
-	sprintf(buf, "0x%x (%s%s%s%s)", flags,
+	sprintf(buf, "0x%x (%s%s%s%s%s)", flags,
 		flags & LCK_PARTIAL_MODE	  ? "PARTIAL_MODE " : "",
 		flags & LCK_MIRROR_NOSYNC_MODE	  ? "MIRROR_NOSYNC " : "",
 		flags & LCK_DMEVENTD_MONITOR_MODE ? "DMEVENTD_MONITOR " : "",
+		flags & LCK_VG_CACHE ? "CACHE " : "",
 		flags & LCK_CONVERT ? "CONVERT " : "");
 
 	return buf;
@@ -680,6 +681,7 @@ static void drop_vg_locks()
  */
 void do_lock_vg(unsigned char command, unsigned char lock_flags, char *resource)
 {
+	uint32_t lock_cmd = command;
 	char *vgname = resource + 2;
 
 	DEBUGLOG("do_lock_vg: resource '%s', cmd = %s, flags = %s, memlock = %d\n",
@@ -691,9 +693,26 @@ void do_lock_vg(unsigned char command, unsigned char lock_flags, char *resource)
 		return;
 	}
 
+	/* Decode command back... */
+	lock_cmd &= (LCK_SCOPE_MASK | LCK_TYPE_MASK | LCK_HOLD);
+	if (lock_flags & LCK_VG_CACHE)
+		lock_cmd |= LCK_CACHE;
+
 	pthread_mutex_lock(&lvm_lock);
-	DEBUGLOG("Dropping metadata for VG %s\n", vgname);
-	lvmcache_drop_metadata(vgname, 0);
+	switch (lock_cmd) {
+		case LCK_VG_COMMIT:
+			DEBUGLOG("vg_commit notification for VG %s\n", vgname);
+			lvmcache_commit_metadata(vgname);
+			break;
+		case LCK_VG_REVERT:
+			DEBUGLOG("vg_revert notification for VG %s\n", vgname);
+			lvmcache_drop_metadata(vgname, 1);
+			break;
+		case LCK_VG_DROP_CACHE:
+		default:
+			DEBUGLOG("Dropping metadata for VG %s\n", vgname);
+			lvmcache_drop_metadata(vgname, 0);
+	}
 	pthread_mutex_unlock(&lvm_lock);
 }
 
lib/locking/cluster_locking.c +3 -1
index 225795a..cf858be 100644
@@ -330,6 +330,9 @@ static int _lock_for_cluster(struct cmd_context *cmd, unsigned char clvmd_cmd,
 	if (cmd->partial_activation)
 		args[1] |= LCK_PARTIAL_MODE;
 
+	if (flags & LCK_CACHE)
+		args[1] |= LCK_VG_CACHE;
+
 	/*
 	 * VG locks are just that: locks, and have no side effects
 	 * so we only need to do them on the local node because all
@@ -409,7 +412,6 @@ int lock_resource(struct cmd_context *cmd, const char *resource, uint32_t flags)
 
 		lock_scope = "VG";
 		clvmd_cmd = CLVMD_CMD_LOCK_VG;
-		flags &= LCK_TYPE_MASK;
 		break;
 
 	case LCK_LV:
lib/locking/locking.h +12 -0
index 50101d1..75325e7 100644
@@ -82,6 +82,8 @@ int check_lvm1_vg_inactive(struct cmd_context *cmd, const char *vgname);
 #define LCK_HOLD	0x00000020U	/* Hold lock when lock_vol returns? */
 #define LCK_LOCAL	0x00000040U	/* Don't propagate to other nodes */
 #define LCK_CLUSTER_VG	0x00000080U	/* VG is clustered */
+/* FIXME: only lowest 8bits send in cluster locking,
+ * in remote node LCK_VG_CACHE is set instead*/
 #define LCK_CACHE	0x00000100U	/* Operation on cache only using P_ lock */
 
 /*
@@ -91,6 +93,7 @@ int check_lvm1_vg_inactive(struct cmd_context *cmd, const char *vgname);
 #define LCK_MIRROR_NOSYNC_MODE	0x00000002U	/* Mirrors don't require sync */
 #define LCK_DMEVENTD_MONITOR_MODE	0x00000004U	/* Register with dmeventd */
 #define LCK_CONVERT		0x00000008U	/* Convert existing lock */
+#define LCK_VG_CACHE		0x00000010U	/* set when LCK_CACHE is set */
 
 /*
  * Special cases of VG locks.
@@ -107,6 +110,11 @@ int check_lvm1_vg_inactive(struct cmd_context *cmd, const char *vgname);
 #define LCK_VG_WRITE		(LCK_VG | LCK_WRITE | LCK_HOLD)
 #define LCK_VG_UNLOCK		(LCK_VG | LCK_UNLOCK)
 #define LCK_VG_DROP_CACHE	(LCK_VG | LCK_WRITE | LCK_CACHE)
+
+/* FIXME: LCK_HOLD abused here */
+#define LCK_VG_COMMIT		(LCK_VG | LCK_WRITE | LCK_CACHE | LCK_HOLD)
+#define LCK_VG_REVERT		(LCK_VG | LCK_READ  | LCK_CACHE | LCK_HOLD)
+
 #define LCK_VG_BACKUP		(LCK_VG | LCK_CACHE)
 
 #define LCK_LV_EXCLUSIVE	(LCK_LV | LCK_EXCL)
@@ -142,6 +150,10 @@ int check_lvm1_vg_inactive(struct cmd_context *cmd, const char *vgname);
 	lock_lv_vol(cmd, lv, LCK_LV_DEACTIVATE | LCK_LOCAL)
 #define drop_cached_metadata(vg)	\
 	lock_vol((vg)->cmd, (vg)->name, LCK_VG_DROP_CACHE)
+#define remote_commit_cached_metadata(vg)	\
+	lock_vol((vg)->cmd, (vg)->name, LCK_VG_COMMIT)
+#define remote_revert_cached_metadata(vg)	\
+	lock_vol((vg)->cmd, (vg)->name, LCK_VG_REVERT)
 #define remote_backup_metadata(vg)	\
 	lock_vol((vg)->cmd, (vg)->name, LCK_VG_BACKUP)
 
lib/metadata/metadata.c +8 -0
index c24bbda..1baa4b6 100644
@@ -2331,6 +2331,12 @@ int vg_commit(struct volume_group *vg)
 		}
 	}
 
+	/*
+	 * Inform that remote nodes so the can drop precommited-only flag.
+	 */
+	if (cache_updated)
+		remote_commit_cached_metadata(vg);
+
 	/* If update failed, remove any cached precommitted metadata. */
 	if (!cache_updated && !drop_cached_metadata(vg))
 		log_error("Attempt to drop cached metadata failed "
@@ -2356,6 +2362,8 @@ int vg_revert(struct volume_group *vg)
 		log_error("Attempt to drop cached metadata failed "
 			  "after reverted update for VG %s.", vg->name);
 
+	remote_revert_cached_metadata(vg);
+
 	return 1;
 }
 
Signature · 1 line
1.6.5.7