Properly decode flags even for VG locks.
PATCH 1/4 Properly decode flags even for VG locks.
And decode flags in humar readable form in client. And clean some trailing whitespaces.
No eal change n this patch (only debugging messages changed).
--- daemons/clvmd/lvm-functions.c | 71 +++++++++++++++++++++-------------------- lib/locking/cluster_locking.c | 13 ++++--- 2 files changed, 43 insertions(+), 41 deletions(-)
daemons/clvmd/lvm-functions.c +36 -35
@@ -74,23 +74,23 @@ static const char *decode_locking_cmd(unsigned char cmdl)const char *command;switch (cmdl &LCK_TYPE_MASK ) { -case LCK_NULL : - type ="NULL" ; +case LCK_NULL : + type ="NULL" ;break ; -case LCK_READ : - type ="READ" ; +case LCK_READ : + type ="READ" ;break ; -case LCK_PREAD : - type ="PREAD" ; +case LCK_PREAD : + type ="PREAD" ;break ; -case LCK_WRITE : - type ="WRITE" ; +case LCK_WRITE : + type ="WRITE" ;break ; -case LCK_EXCL : - type ="EXCL" ; +case LCK_EXCL : + type ="EXCL" ;break ; -case LCK_UNLOCK : - type ="UNLOCK" ; +case LCK_UNLOCK : + type ="UNLOCK" ;break ;default : type ="unknown" ;
@@ -98,34 +98,35 @@ static const char *decode_locking_cmd(unsigned char cmdl) }switch (cmdl &LCK_SCOPE_MASK ) { -case LCK_VG : - scope ="VG" ; +case LCK_VG : + scope ="VG" ; + command ="LCK_VG" ;break ; -case LCK_LV : +case LCK_LV : scope ="LV" ; +switch (cmdl &LCK_MASK ) { +case LCK_LV_EXCLUSIVE &LCK_MASK : + command ="LCK_LV_EXCLUSIVE" ; +break ; +case LCK_LV_SUSPEND &LCK_MASK : + command ="LCK_LV_SUSPEND" ; +break ; +case LCK_LV_RESUME &LCK_MASK : + command ="LCK_LV_RESUME" ; +break ; +case LCK_LV_ACTIVATE &LCK_MASK : + command ="LCK_LV_ACTIVATE" ; +break ; +case LCK_LV_DEACTIVATE &LCK_MASK : + command ="LCK_LV_DEACTIVATE" ; +break ; +default : + command ="unknown" ; +break ; + }break ;default : scope ="unknown" ; -break ; - } - -switch (cmdl &LCK_MASK ) { -case LCK_LV_EXCLUSIVE &LCK_MASK : - command ="LCK_LV_EXCLUSIVE" ; -break ; -case LCK_LV_SUSPEND &LCK_MASK : - command ="LCK_LV_SUSPEND" ; -break ; -case LCK_LV_RESUME &LCK_MASK : - command ="LCK_LV_RESUME" ; -break ; -case LCK_LV_ACTIVATE &LCK_MASK : - command ="LCK_LV_ACTIVATE" ; -break ; -case LCK_LV_DEACTIVATE &LCK_MASK : - command ="LCK_LV_DEACTIVATE" ; -break ; -default : command ="unknown" ;break ; }
lib/locking/cluster_locking.c +7 -6
@@ -450,12 +450,13 @@ int lock_resource(struct cmd_context *cmd, const char *resource, uint32_t flags)return 0 ; } -log_very_verbose ("Locking %s %s %s %s%s%s%s (0x%x)" , lock_scope, lockname, - lock_type, - flags &LCK_NONBLOCK ?"" :"B" , - flags &LCK_HOLD ?"H" :"" , - flags &LCK_LOCAL ?"L" :"" , - flags &LCK_CLUSTER_VG ?"C" :"" , +log_very_verbose ("Locking %s %s %s (%s%s%s%s%s%s) (0x%x)" , lock_scope, lockname, + lock_type, lock_scope, + flags &LCK_NONBLOCK ?"|NONBLOCK" :"" , + flags &LCK_HOLD ?"|HOLD" :"" , + flags &LCK_LOCAL ?"|LOCAL" :"" , + flags &LCK_CLUSTER_VG ?"|CLUSTER" :"" , + flags &LCK_CACHE ?"|CACHE" :"" , flags);/* Send a message to the cluster manager */
Signature · 1 line
1.6.5.7
PATCH 2/4 Move processing of VG locks to separate function (similar to LV locks).
And print some debugging info.
No functional change in this patch.
--- daemons/clvmd/clvmd-command.c | 9 +++------ daemons/clvmd/lvm-functions.c | 17 ++++++++++++++--- daemons/clvmd/lvm-functions.h | 3 ++- 3 files changed, 19 insertions(+), 10 deletions(-)
daemons/clvmd/clvmd-command.c +3 -6
@@ -119,15 +119,12 @@ int do_command(struct local_client *client, struct clvm_header *msg, int msglen,break ;case CLVMD_CMD_LOCK_VG : + lock_cmd = args[0 ]; + lock_flags = args[1 ]; lockname = &args[2 ];/* Check to see if the VG is in use by LVM1 */ status =do_check_lvm1 (lockname); -/* P_#global causes a full cache refresh */ -if (!strcmp (lockname,"P_" VG_GLOBAL )) -do_refresh_cache (); -else -drop_metadata (lockname +2 ); - +do_lock_vg (lock_cmd, lock_flags, lockname);break ;case CLVMD_CMD_LOCK_LV :
daemons/clvmd/lvm-functions.c +14 -3
@@ -676,12 +676,23 @@ static void drop_vg_locks() }/* -* Drop lvmcache metadata +* Handle VG lock - drop metadata or update lvmcache state */ -void drop_metadata (const char *vgname) +void do_lock_vg (unsigned char command,unsigned char lock_flags,char *resource) { -DEBUGLOG ("Dropping metadata for VG %s\n" , vgname); +char *vgname = resource +2 ; + +DEBUGLOG ("do_lock_vg: resource '%s', cmd = %s, flags = %s, memlock = %d\n" , + resource,decode_locking_cmd (command),decode_flags (lock_flags),memlock ()); + +/* P_#global causes a full cache refresh */ +if (!strcmp (resource,"P_" VG_GLOBAL )) { +do_refresh_cache (); +return ; + } +pthread_mutex_lock (&lvm_lock); +DEBUGLOG ("Dropping metadata for VG %s\n" , vgname);lvmcache_drop_metadata (vgname);pthread_mutex_unlock (&lvm_lock); }
daemons/clvmd/lvm-functions.h +2 -1
@@ -35,6 +35,7 @@ extern void lvm_do_backup(const char *vgname);extern int hold_unlock (char *resource);extern int hold_lock (char *resource,int mode,int flags);extern char *get_last_lvm_error (void ); -extern void drop_metadata (const char *vgname); +extern void do_lock_vg (unsigned char command,unsigned char lock_flags, +char *resource);#endif
Signature · 1 line
1.6.5.7
PATCH 3/4 Add possibility to handle precommitted metadata in lvmcache.
In reply to Milan Broz - Add drop_precommitted flag to force drop precommited metadata
- add lvmcache_commit_metadata() which upgrades precommitted metadata in cache
No functional change in this patch - just preparation for following change.
--- daemons/clvmd/lvm-functions.c | 2 +- lib/cache/lvmcache.c | 27 ++++++++++++++++++++------- lib/cache/lvmcache.h | 3 ++- lib/locking/file_locking.c | 2 +- 4 files changed, 24 insertions(+), 10 deletions(-)
daemons/clvmd/lvm-functions.c +1 -1
@@ -693,7 +693,7 @@ void do_lock_vg(unsigned char command, unsigned char lock_flags, char *resource)pthread_mutex_lock (&lvm_lock);DEBUGLOG ("Dropping metadata for VG %s\n" , vgname); -lvmcache_drop_metadata (vgname); +lvmcache_drop_metadata (vgname,0 );pthread_mutex_unlock (&lvm_lock); }
lib/cache/lvmcache.c +20 -7
@@ -158,7 +158,7 @@ static void _update_cache_lock_state(const char *vgname, int locked)_update_cache_vginfo_lock_state (vginfo, locked); } -static void _drop_metadata (const char *vgname) +static void _drop_metadata (const char *vgname,int drop_precommitted) {struct lvmcache_vginfo *vginfo;struct lvmcache_info *info;
@@ -173,25 +173,38 @@ static void _drop_metadata(const char *vgname)* and we must not do it again. */ -if (!vginfo->precommitted) +if (drop_precommitted || !vginfo->precommitted)dm_list_iterate_items (info, &vginfo->infos) info->status |= CACHE_INVALID;_free_cached_vgmetadata (vginfo); } -void lvmcache_drop_metadata (const char *vgname) +/* +* Remote node uses this camm to upgrade precommited metadata to commited state +*/ +void lvmcache_commit_metadata (const char *vgname) +{ +struct lvmcache_vginfo *vginfo; + +if (!(vginfo =vginfo_from_vgname (vgname,NULL ))) +return ; + + vginfo->precommitted =0 ; +} + +void lvmcache_drop_metadata (const char *vgname,int drop_precommitted) {/* For VG_ORPHANS, we need to invalidate all labels on orphan PVs. */ if (!strcmp (vgname,VG_ORPHANS )) { -_drop_metadata (FMT_TEXT_ORPHAN_VG_NAME ); -_drop_metadata (FMT_LVM1_ORPHAN_VG_NAME ); -_drop_metadata (FMT_POOL_ORPHAN_VG_NAME ); +_drop_metadata (FMT_TEXT_ORPHAN_VG_NAME ,0 ); +_drop_metadata (FMT_LVM1_ORPHAN_VG_NAME ,0 ); +_drop_metadata (FMT_POOL_ORPHAN_VG_NAME ,0 );/* Indicate that PVs could now be missing from the cache */ init_full_scan_done (0 ); }else if (!vgname_is_locked (VG_GLOBAL )) -_drop_metadata (vgname); +_drop_metadata (vgname, drop_precommitted); }/*
lib/cache/lvmcache.h +2 -1
@@ -111,6 +111,7 @@ struct dm_list *lvmcache_get_pvids(struct cmd_context *cmd, const char *vgname,/* Returns cached volume group metadata. */ struct volume_group *lvmcache_get_vg (const char *vgid,unsigned precommitted); -void lvmcache_drop_metadata (const char *vgname); +void lvmcache_drop_metadata (const char *vgname,int drop_precommitted); +void lvmcache_commit_metadata (const char *vgname);#endif
lib/locking/file_locking.c +1 -1
@@ -259,7 +259,7 @@ static int _file_lock_resource(struct cmd_context *cmd, const char *resource,case LCK_VG :/* Skip cache refresh for VG_GLOBAL - the caller handles it */ if (strcmp (resource,VG_GLOBAL )) -lvmcache_drop_metadata (resource); +lvmcache_drop_metadata (resource,0 );/* LCK_CACHE does not require a real lock */ if (flags &LCK_CACHE )
Signature · 1 line
1.6.5.7
PATCH 4/4 Propagate commit and revert metadata event to other nodes in cluster.
In reply to Milan BrozThis patch tries to corectly track changes in lvmcache related to commit/revert.
For vg_commit: if there are cached precommitted metadata, after successfull commit these metadata must be tracked as committed.
For vg_revert: remote nodes must drop precommitted metadata and its flag in lvmcache.
Because of current clvmd protocol limitation (unsigned char) there are some non-obvious hacks used.
- The LCK_CACHE bit is translated to another "optional" flag (because we are out of lock flags)
- The LOCK_HOLD (which is normally unused in VG locks) is used in commit/revert message
Also patch removes LCK_TYPES_MASKing which currently prevents these flas to be send to remote nodes.
(N.B. Patch do not touch LV locks here in any way.)
All this machinery is needed to properly solve remote node cache invalidaton which cause several problems recently observed.
--- daemons/clvmd/lvm-functions.c | 25 ++++++++++++++++++++++--- lib/locking/cluster_locking.c | 4 +++- lib/locking/locking.h | 12 ++++++++++++ lib/metadata/metadata.c | 8 ++++++++ 4 files changed, 45 insertions(+), 4 deletions(-)
daemons/clvmd/lvm-functions.c +22 -3
@@ -145,10 +145,11 @@ static const char *decode_flags(unsigned char flags) {static char buf[128 ]; -sprintf (buf,"0x%x (%s%s%s%s)" , flags, +sprintf (buf,"0x%x (%s%s%s%s%s)" , flags, flags &LCK_PARTIAL_MODE ?"PARTIAL_MODE " :"" , flags &LCK_MIRROR_NOSYNC_MODE ?"MIRROR_NOSYNC " :"" , flags &LCK_DMEVENTD_MONITOR_MODE ?"DMEVENTD_MONITOR " :"" , + flags &LCK_VG_CACHE ?"CACHE " :"" , flags &LCK_CONVERT ?"CONVERT " :"" );return buf;
@@ -680,6 +681,7 @@ static void drop_vg_locks()*/ void do_lock_vg (unsigned char command,unsigned char lock_flags,char *resource) { +uint32_t lock_cmd = command;char *vgname = resource +2 ; DEBUGLOG("do_lock_vg: resource '%s', cmd = %s, flags = %s, memlock = %d\n",
@@ -691,9 +693,26 @@ void do_lock_vg(unsigned char command, unsigned char lock_flags, char *resource)return ; } +/* Decode command back... */ + lock_cmd &= (LCK_SCOPE_MASK |LCK_TYPE_MASK |LCK_HOLD ); +if (lock_flags &LCK_VG_CACHE ) + lock_cmd |=LCK_CACHE ; +pthread_mutex_lock (&lvm_lock); -DEBUGLOG ("Dropping metadata for VG %s\n" , vgname); -lvmcache_drop_metadata (vgname,0 ); +switch (lock_cmd) { +case LCK_VG_COMMIT : +DEBUGLOG ("vg_commit notification for VG %s\n" , vgname); +lvmcache_commit_metadata (vgname); +break ; +case LCK_VG_REVERT : +DEBUGLOG ("vg_revert notification for VG %s\n" , vgname); +lvmcache_drop_metadata (vgname,1 ); +break ; +case LCK_VG_DROP_CACHE : +default : +DEBUGLOG ("Dropping metadata for VG %s\n" , vgname); +lvmcache_drop_metadata (vgname,0 ); + }pthread_mutex_unlock (&lvm_lock); }
lib/locking/cluster_locking.c +3 -1
@@ -330,6 +330,9 @@ static int _lock_for_cluster(struct cmd_context *cmd, unsigned char clvmd_cmd,if (cmd->partial_activation) args[1 ] |=LCK_PARTIAL_MODE ; +if (flags &LCK_CACHE ) + args[1 ] |=LCK_VG_CACHE ; +/* * VG locks are just that: locks, and have no side effects * so we only need to do them on the local node because all
@@ -409,7 +412,6 @@ int lock_resource(struct cmd_context *cmd, const char *resource, uint32_t flags) lock_scope ="VG" ; clvmd_cmd =CLVMD_CMD_LOCK_VG ; - flags &=LCK_TYPE_MASK ;break ;case LCK_LV :
lib/locking/locking.h +12 -0
@@ -82,6 +82,8 @@ int check_lvm1_vg_inactive(struct cmd_context *cmd, const char *vgname);#define LCK_HOLD 0x00000020U/* Hold lock when lock_vol returns? */ #define LCK_LOCAL 0x00000040U/* Don't propagate to other nodes */ #define LCK_CLUSTER_VG 0x00000080U/* VG is clustered */ +/* FIXME: only lowest 8bits send in cluster locking, +* in remote node LCK_VG_CACHE is set instead*/ #define LCK_CACHE 0x00000100U/* Operation on cache only using P_ lock */ /*
@@ -91,6 +93,7 @@ int check_lvm1_vg_inactive(struct cmd_context *cmd, const char *vgname);#define LCK_MIRROR_NOSYNC_MODE 0x00000002U/* Mirrors don't require sync */ #define LCK_DMEVENTD_MONITOR_MODE 0x00000004U/* Register with dmeventd */ #define LCK_CONVERT 0x00000008U/* Convert existing lock */ +#define LCK_VG_CACHE 0x00000010U/* set when LCK_CACHE is set */ /* * Special cases of VG locks.
@@ -107,6 +110,11 @@ int check_lvm1_vg_inactive(struct cmd_context *cmd, const char *vgname);#define LCK_VG_WRITE (LCK_VG | LCK_WRITE | LCK_HOLD)#define LCK_VG_UNLOCK (LCK_VG | LCK_UNLOCK)#define LCK_VG_DROP_CACHE (LCK_VG | LCK_WRITE | LCK_CACHE) + +/* FIXME: LCK_HOLD abused here */ +#define LCK_VG_COMMIT (LCK_VG | LCK_WRITE | LCK_CACHE | LCK_HOLD) +#define LCK_VG_REVERT (LCK_VG | LCK_READ | LCK_CACHE | LCK_HOLD) +#define LCK_VG_BACKUP (LCK_VG | LCK_CACHE)#define LCK_LV_EXCLUSIVE (LCK_LV | LCK_EXCL)
@@ -142,6 +150,10 @@ int check_lvm1_vg_inactive(struct cmd_context *cmd, const char *vgname); lock_lv_vol(cmd, lv, LCK_LV_DEACTIVATE | LCK_LOCAL)#define drop_cached_metadata (vg) \ lock_vol((vg)->cmd, (vg)->name, LCK_VG_DROP_CACHE) +#define remote_commit_cached_metadata (vg) \ + lock_vol((vg)->cmd, (vg)->name, LCK_VG_COMMIT) +#define remote_revert_cached_metadata (vg) \ + lock_vol((vg)->cmd, (vg)->name, LCK_VG_REVERT)#define remote_backup_metadata (vg) \ lock_vol((vg)->cmd, (vg)->name, LCK_VG_BACKUP)
lib/metadata/metadata.c +8 -0
@@ -2331,6 +2331,12 @@ int vg_commit(struct volume_group *vg) } } +/* +* Inform that remote nodes so the can drop precommited-only flag. +*/ +if (cache_updated) +remote_commit_cached_metadata (vg); +/* If update failed, remove any cached precommitted metadata. */ if (!cache_updated && !drop_cached_metadata (vg)) log_error("Attempt to drop cached metadata failed "
@@ -2356,6 +2362,8 @@ int vg_revert(struct volume_group *vg)log_error ("Attempt to drop cached metadata failed " "after reverted update for VG %s." , vg->name); +remote_revert_cached_metadata (vg); +return 1 ; }
Signature · 1 line
1.6.5.7