Convert mle list to a hash
These patches convert the mle list to a hash. The same patches apply on ocfs2 1.4 too.
Currently, we use the same number of hash pages for mles and lockres'. This will be addressed in a future patch that will make both of them configurable.
Sunil
PATCH 1/10 ocfs2/dlm: Encapsulate adding and removing of mle from dlm->master_list
This patch encapsulates adding and removing of the mle from the dlm->master_list. This patch is part of the series of patches that converts the mle list to a mle hash.
--- fs/ocfs2/dlm/dlmcommon.h | 3 +++ fs/ocfs2/dlm/dlmmaster.c | 34 +++++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 11 deletions(-)
fs/ocfs2/dlm/dlmcommon.h +3 -0
@@ -1008,6 +1008,9 @@ static inline void __dlm_wait_on_lockres(struct dlm_lock_resource *res) DLM_LOCK_RES_MIGRATING)); } +void __dlm_unlink_mle (struct dlm_ctxt *dlm,struct dlm_master_list_entry *mle); +void __dlm_insert_mle (struct dlm_ctxt *dlm,struct dlm_master_list_entry *mle); +/* create/destroy slab caches */ int dlm_init_master_caches (void );void dlm_destroy_master_caches (void );
fs/ocfs2/dlm/dlmmaster.c +23 -11
@@ -318,6 +318,21 @@ static void dlm_init_mle(struct dlm_master_list_entry *mle,__dlm_mle_attach_hb_events (dlm, mle); } +void __dlm_unlink_mle (struct dlm_ctxt *dlm,struct dlm_master_list_entry *mle) +{ +assert_spin_locked (&dlm->spinlock); +assert_spin_locked (&dlm->master_lock); + +if (!list_empty (&mle->list)) +list_del_init (&mle->list); +} + +void __dlm_insert_mle (struct dlm_ctxt *dlm,struct dlm_master_list_entry *mle) +{ +assert_spin_locked (&dlm->master_lock); + +list_add (&mle->list, &dlm->master_list); +}/* returns 1 if found, 0 if not */ static int dlm_find_mle(struct dlm_ctxt *dlm,
@@ -420,8 +435,7 @@ static void dlm_mle_release(struct kref *kref)assert_spin_locked (&dlm->master_lock);/* remove from list if not already */ -if (!list_empty (&mle->list)) -list_del_init (&mle->list); +__dlm_unlink_mle (dlm, mle);/* detach the mle from the domain node up/down events */ __dlm_mle_detach_hb_events (dlm, mle);
@@ -843,7 +857,7 @@ lookup: alloc_mle =NULL ;dlm_init_mle (mle,DLM_MLE_MASTER , dlm, res,NULL ,0 );set_bit (dlm->node_num, mle->maybe_map); -list_add (&mle->list, &dlm->master_list); +__dlm_insert_mle (dlm, mle);/* still holding the dlm spinlock, check the recovery map * to see if there are any nodes that still need to be
@@ -1575,7 +1589,7 @@ way_up_top:// "add the block.\n"); dlm_init_mle (mle,DLM_MLE_BLOCK , dlm,NULL , name, namelen);set_bit (request->node_idx, mle->maybe_map); -list_add (&mle->list, &dlm->master_list); +__dlm_insert_mle (dlm, mle); response =DLM_MASTER_RESP_NO ; }else {// mlog(0, "mle was found\n");
@@ -1967,7 +1981,7 @@ ok: assert->node_idx, rr, extra_ref, mle->inuse);dlm_print_one_mle (mle); } -list_del_init (&mle->list); +__dlm_unlink_mle (dlm, mle);__dlm_mle_detach_hb_events (dlm, mle);__dlm_put_mle (mle);if (extra_ref) {
@@ -3159,10 +3173,8 @@ static int dlm_add_migration_mle(struct dlm_ctxt *dlm, tmp->master = master;atomic_set (&tmp->woken,1 );wake_up (&tmp->wq); -/* remove it from the list so that only one -* mle will be found */ -list_del_init (&tmp->list); -/* this was obviously WRONG. mle is uninited here. should be tmp. */ +/* remove it so that only one mle will be found */ +__dlm_unlink_mle (dlm, tmp);__dlm_mle_detach_hb_events (dlm, tmp); ret =DLM_MIGRATE_RESPONSE_MASTERY_REF ; mlog(0, "%s:%.*s: master=%u, newmaster=%u, "
@@ -3181,7 +3193,7 @@ static int dlm_add_migration_mle(struct dlm_ctxt *dlm, mle->master = master;/* do this for consistency with other mle types */ set_bit (new_master, mle->maybe_map); -list_add (&mle->list, &dlm->master_list); +__dlm_insert_mle (dlm, mle);return ret; }
@@ -3264,7 +3276,7 @@ top:* list_head while in list_for_each_safe */ __dlm_mle_detach_hb_events (dlm, mle);spin_lock (&mle->spinlock); -list_del_init (&mle->list); +__dlm_unlink_mle (dlm, mle);atomic_set (&mle->woken,1 );spin_unlock (&mle->spinlock);wake_up (&mle->wq);
Signature · 1 line
1.5.6.3
PATCH 1/10 ocfs2/dlm: Encapsulate adding and removing of mle from dlm->master_list
Quoted 5 lines · Sunil Mushran
This patch encapsulates adding and removing of the mle from the
dlm->master_list. This patch is part of the series of patches that converts the mle list to a mle hash.Signed-off-by Sunil Mushran <[email protected]>
sob
Quoted 121 lines
--- fs/ocfs2/dlm/dlmcommon.h | 3 +++ fs/ocfs2/dlm/dlmmaster.c | 34 +++++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 11 deletions(-)fs/ocfs2/dlm/dlmcommon.h +3
-0@@ -1008,6 +1008,9 @@ static inline void __dlm_wait_on_lockres(struct dlm_lock_resource *res) DLM_LOCK_RES_MIGRATING)); } +void __dlm_unlink_mle (struct dlm_ctxt *dlm,struct dlm_master_list_entry *mle); +void __dlm_insert_mle (struct dlm_ctxt *dlm,struct dlm_master_list_entry *mle); +/* create/destroy slab caches */ int dlm_init_master_caches (void );void dlm_destroy_master_caches (void );fs/ocfs2/dlm/dlmmaster.c +23
-11@@ -318,6 +318,21 @@ static void dlm_init_mle(struct dlm_master_list_entry *mle,__dlm_mle_attach_hb_events (dlm, mle); } +void __dlm_unlink_mle (struct dlm_ctxt *dlm,struct dlm_master_list_entry *mle) +{ +assert_spin_locked (&dlm->spinlock); +assert_spin_locked (&dlm->master_lock); + +if (!list_empty (&mle->list)) +list_del_init (&mle->list); +} + +void __dlm_insert_mle (struct dlm_ctxt *dlm,struct dlm_master_list_entry *mle) +{ +assert_spin_locked (&dlm->master_lock); + +list_add (&mle->list, &dlm->master_list); +}/* returns 1 if found, 0 if not */ static int dlm_find_mle(struct dlm_ctxt *dlm,@@ -420,8 +435,7 @@ static void dlm_mle_release(struct kref *kref)assert_spin_locked (&dlm->master_lock);/* remove from list if not already */ -if (!list_empty (&mle->list)) -list_del_init (&mle->list); +__dlm_unlink_mle (dlm, mle);/* detach the mle from the domain node up/down events */ __dlm_mle_detach_hb_events (dlm, mle);@@ -843,7 +857,7 @@ lookup: alloc_mle =NULL ;dlm_init_mle (mle,DLM_MLE_MASTER , dlm, res,NULL ,0 );set_bit (dlm->node_num, mle->maybe_map); -list_add (&mle->list, &dlm->master_list); +__dlm_insert_mle (dlm, mle);/* still holding the dlm spinlock, check the recovery map * to see if there are any nodes that still need to be @@ -1575,7 +1589,7 @@ way_up_top:// "add the block.\n"); dlm_init_mle (mle,DLM_MLE_BLOCK , dlm,NULL , name, namelen);set_bit (request->node_idx, mle->maybe_map); -list_add (&mle->list, &dlm->master_list); +__dlm_insert_mle (dlm, mle); response =DLM_MASTER_RESP_NO ; }else {// mlog(0, "mle was found\n"); @@ -1967,7 +1981,7 @@ ok: assert->node_idx, rr, extra_ref, mle->inuse);dlm_print_one_mle (mle); } -list_del_init (&mle->list); +__dlm_unlink_mle (dlm, mle);__dlm_mle_detach_hb_events (dlm, mle);__dlm_put_mle (mle);if (extra_ref) {@@ -3159,10 +3173,8 @@ static int dlm_add_migration_mle(struct dlm_ctxt *dlm, tmp->master = master;atomic_set (&tmp->woken,1 );wake_up (&tmp->wq); -/* remove it from the list so that only one -* mle will be found */ -list_del_init (&tmp->list); -/* this was obviously WRONG. mle is uninited here. should be tmp. */ +/* remove it so that only one mle will be found */ +__dlm_unlink_mle (dlm, tmp);__dlm_mle_detach_hb_events (dlm, tmp); ret =DLM_MIGRATE_RESPONSE_MASTERY_REF ; mlog(0, "%s:%.*s: master=%u, newmaster=%u, "@@ -3181,7 +3193,7 @@ static int dlm_add_migration_mle(struct dlm_ctxt *dlm, mle->master = master;/* do this for consistency with other mle types */ set_bit (new_master, mle->maybe_map); -list_add (&mle->list, &dlm->master_list); +__dlm_insert_mle (dlm, mle);return ret; }@@ -3264,7 +3276,7 @@ top:* list_head while in list_for_each_safe */ __dlm_mle_detach_hb_events (dlm, mle);spin_lock (&mle->spinlock); -list_del_init (&mle->list); +__dlm_unlink_mle (dlm, mle);atomic_set (&mle->woken,1 );spin_unlock (&mle->spinlock);wake_up (&mle->wq);Signature · 7 lines
1.5.6.3 _______________________________________________ Ocfs2-devel mailing list Ocfs2-devel at oss.oracle.com http://oss.oracle.com/mailman/listinfo/ocfs2-devel
Signature · 8 lines
"We'd better get back, `cause it'll be dark soon, and they mostly come at night. Mostly." Joel Becker Principal Software Developer Oracle E-mail: joel.becker at oracle.com Phone: (650) 506-8127
PATCH 2/10 ocfs2/dlm: Clean up struct dlm_lock_name
In reply to Sunil MushranFor master mle, the name it stored in the attached lockres in struct qstr. For block and migration mle, the name is stored inline in struct dlm_lock_name. This patch attempts to make struct dlm_lock_name look like a struct qstr. While we could use struct qstr, we don't because we want to avoid having to malloc and free the lockname string as the mle's lifetime is fairly short.
--- fs/ocfs2/dlm/dlmcommon.h | 8 ++-- fs/ocfs2/dlm/dlmdebug.c | 10 +++--- fs/ocfs2/dlm/dlmmaster.c | 80 ++++++++++++++++++++++++++-------------------- 3 files changed, 54 insertions(+), 44 deletions(-)
fs/ocfs2/dlm/dlmcommon.h +4 -4
@@ -56,8 +56,8 @@ enum dlm_mle_type { };struct dlm_lock_name { -u8 len; -u8 name[DLM_LOCKID_NAME_MAX ]; +unsigned int len; +unsigned char name[DLM_LOCKID_NAME_MAX ]; };struct dlm_master_list_entry {
@@ -79,8 +79,8 @@ struct dlm_master_list_entry {struct o2hb_callback_func mle_hb_up;struct o2hb_callback_func mle_hb_down;union { -struct dlm_lock_resource *res; -struct dlm_lock_name name; +struct dlm_lock_resource *mleres; +struct dlm_lock_name mlename; } u; };
fs/ocfs2/dlm/dlmdebug.c +5 -5
@@ -288,15 +288,15 @@ static int dump_mle(struct dlm_master_list_entry *mle, char *buf, int len) {int out =0 ;unsigned int namelen; -const char *name; +unsigned char *name;char *mle_type;if (mle->type !=DLM_MLE_MASTER ) { - namelen = mle->u.name.len; - name = mle->u.name.name; + name = mle->u.mlename.name; + namelen = mle->u.mlename.len; }else { - namelen = mle->u.res->lockname.len; - name = mle->u.res->lockname.name; + name = (unsigned char *)mle->u.mleres->lockname.name; + namelen = mle->u.mleres->lockname.len; }if (mle->type ==DLM_MLE_BLOCK )
fs/ocfs2/dlm/dlmmaster.c +45 -35
136 lines not on this page · raw message
Signature · 1 line
1.5.6.3
PATCH 3/10 ocfs2/dlm: Refactor dlm_clean_master_list()
In reply to Sunil MushranThis patch refactors dlm_clean_master_list() so as to make it easier to convert the mle list to a hash.
--- fs/ocfs2/dlm/dlmmaster.c | 148 ++++++++++++++++++++++++++------------------- 1 files changed, 85 insertions(+), 63 deletions(-)
fs/ocfs2/dlm/dlmmaster.c +85 -63
@@ -3208,12 +3208,87 @@ static int dlm_add_migration_mle(struct dlm_ctxt *dlm,return ret; } +/* +* Sets the owner of the lockres, associated to the mle, to UNKNOWN +*/ +static struct dlm_lock_resource *dlm_reset_mleres_owner (struct dlm_ctxt *dlm, +struct dlm_master_list_entry *mle) +{ +struct dlm_lock_resource *res; +unsigned int hash; + +/* Find the lockres associated to the mle and set its owner to UNK */ + hash =dlm_lockid_hash (mle->u.mlename.name, mle->u.mlename.len); + res =__dlm_lookup_lockres (dlm, mle->u.mlename.name, mle->u.mlename.len, + hash); +if (res) { +spin_unlock (&dlm->master_lock); + +/* move lockres onto recovery list */ +spin_lock (&res->spinlock); +dlm_set_lockres_owner (dlm, res,DLM_LOCK_RES_OWNER_UNKNOWN ); +dlm_move_lockres_to_recovery_list (dlm, res); +spin_unlock (&res->spinlock); +dlm_lockres_put (res); + +/* about to get rid of mle, detach from heartbeat */ +__dlm_mle_detach_hb_events (dlm, mle); + +/* dump the mle */ +spin_lock (&dlm->master_lock); +__dlm_put_mle (mle); +spin_unlock (&dlm->master_lock); + } + +return res; +} + +static void dlm_clean_migration_mle (struct dlm_ctxt *dlm, +struct dlm_master_list_entry *mle) +{ +__dlm_mle_detach_hb_events (dlm, mle); + +spin_lock (&mle->spinlock); +__dlm_unlink_mle (dlm, mle); +atomic_set (&mle->woken,1 ); +spin_unlock (&mle->spinlock); + +wake_up (&mle->wq); +} + +static void dlm_clean_block_mle (struct dlm_ctxt *dlm, +struct dlm_master_list_entry *mle,u8 dead_node) +{ +int bit; + +BUG_ON (mle->type !=DLM_MLE_BLOCK ); + +spin_lock (&mle->spinlock); + bit =find_next_bit (mle->maybe_map,O2NM_MAX_NODES ,0 ); +if (bit != dead_node) { +mlog (0 ,"mle found, but dead node %u would not have been " +"master\n" , dead_node); +spin_unlock (&mle->spinlock); + }else { +/* Must drop the refcount by one since the assert_master will +* never arrive. This may result in the mle being unlinked and +* freed, but there may still be a process waiting in the +* dlmlock path which is fine. */ +mlog (0 ,"node %u was expected master\n" , dead_node); +atomic_set (&mle->woken,1 ); +spin_unlock (&mle->spinlock); +wake_up (&mle->wq); + +/* Do not need events any longer, so detach from heartbeat */ +__dlm_mle_detach_hb_events (dlm, mle); +__dlm_put_mle (mle); + } +}void dlm_clean_master_list (struct dlm_ctxt *dlm,u8 dead_node) {struct dlm_master_list_entry *mle, *next;struct dlm_lock_resource *res; -unsigned int hash;mlog_entry ("dlm=%s, dead node=%u\n" , dlm->name, dead_node); top:
@@ -3237,30 +3312,7 @@ top:* need to clean up if the dead node would have * been the master. */ if (mle->type ==DLM_MLE_BLOCK ) { -int bit; - -spin_lock (&mle->spinlock); - bit =find_next_bit (mle->maybe_map,O2NM_MAX_NODES ,0 ); -if (bit != dead_node) { -mlog (0 ,"mle found, but dead node %u would " -"not have been master\n" , dead_node); -spin_unlock (&mle->spinlock); - }else { -/* must drop the refcount by one since the -* assert_master will never arrive. this -* may result in the mle being unlinked and -* freed, but there may still be a process -* waiting in the dlmlock path which is fine. */ -mlog (0 ,"node %u was expected master\n" , - dead_node); -atomic_set (&mle->woken,1 ); -spin_unlock (&mle->spinlock); -wake_up (&mle->wq); -/* do not need events any longer, so detach -* from heartbeat */ -__dlm_mle_detach_hb_events (dlm, mle); -__dlm_put_mle (mle); - } +dlm_clean_block_mle (dlm, mle, dead_node);continue ; }
@@ -3281,51 +3333,21 @@ top:/* if we have reached this point, this mle needs to * be removed from the list and freed. */ - -/* remove from the list early. NOTE: unlinking -* list_head while in list_for_each_safe */ -__dlm_mle_detach_hb_events (dlm, mle); -spin_lock (&mle->spinlock); -__dlm_unlink_mle (dlm, mle); -atomic_set (&mle->woken,1 ); -spin_unlock (&mle->spinlock); -wake_up (&mle->wq); +dlm_clean_migration_mle (dlm, mle);mlog (0 ,"%s: node %u died during migration from " "%u to %u!\n" , dlm->name, dead_node, mle->master, mle->new_master); -/* if there is a lockres associated with this -* mle, find it and set its owner to UNKNOWN */ - hash =dlm_lockid_hash (mle->u.mlename.name, mle->u.mlename.len); - res =__dlm_lookup_lockres (dlm, mle->u.mlename.name, - mle->u.mlename.len, hash); -if (res) { -/* unfortunately if we hit this rare case, our -* lock ordering is messed. we need to drop -* the master lock so that we can take the -* lockres lock, meaning that we will have to -* restart from the head of list. */ -spin_unlock (&dlm->master_lock); - -/* move lockres onto recovery list */ -spin_lock (&res->spinlock); -dlm_set_lockres_owner (dlm, res, -DLM_LOCK_RES_OWNER_UNKNOWN ); -dlm_move_lockres_to_recovery_list (dlm, res); -spin_unlock (&res->spinlock); -dlm_lockres_put (res); - -/* about to get rid of mle, detach from heartbeat */ -__dlm_mle_detach_hb_events (dlm, mle); - -/* dump the mle */ -spin_lock (&dlm->master_lock); -__dlm_put_mle (mle); -spin_unlock (&dlm->master_lock); +/* If we find a lockres associated with the mle, we've +* hit this rare case that messes up our lock ordering. +* If so, we need to drop the master lock so that we can +* take the lockres lock, meaning that we will have to +* restart from the head of list. */ + res =dlm_reset_mleres_owner (dlm, mle); +if (res)/* restart */ goto top; - }/* this may be the last reference */ __dlm_put_mle (mle);
Signature · 1 line
1.5.6.3
PATCH 4/10 ocfs2/dlm: Create and destroy the dlm->master_hash
In reply to Sunil MushranThis patch adds code to create and destroy the dlm->master_hash.
--- fs/ocfs2/dlm/dlmcommon.h | 8 ++++++++ fs/ocfs2/dlm/dlmdomain.c | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 0 deletions(-)
fs/ocfs2/dlm/dlmcommon.h +8 -0
@@ -151,6 +151,7 @@ struct dlm_ctxtunsigned long recovery_map[BITS_TO_LONGS (O2NM_MAX_NODES )];struct dlm_recovery_ctxt reco;spinlock_t master_lock; +struct hlist_head **master_hash;struct list_head master_list;struct list_head mle_hb_events;
@@ -195,6 +196,13 @@ static inline struct hlist_head *dlm_lockres_hash(struct dlm_ctxt *dlm, unsignedreturn dlm->lockres_hash[(i /DLM_BUCKETS_PER_PAGE ) %DLM_HASH_PAGES ] + (i %DLM_BUCKETS_PER_PAGE ); } +static inline struct hlist_head *dlm_master_hash (struct dlm_ctxt *dlm, +unsigned i) +{ +return dlm->master_hash[(i /DLM_BUCKETS_PER_PAGE ) %DLM_HASH_PAGES ] + + (i %DLM_BUCKETS_PER_PAGE ); +} +/* these keventd work queue items are for less-frequently * called functions that cannot be directly called from the * net message handlers for some reason, usually because
fs/ocfs2/dlm/dlmdomain.c +18 -0
@@ -304,6 +304,9 @@ static void dlm_free_ctxt_mem(struct dlm_ctxt *dlm)if (dlm->lockres_hash)dlm_free_pagevec ((void **)dlm->lockres_hash,DLM_HASH_PAGES ); +if (dlm->master_hash) +dlm_free_pagevec ((void **)dlm->master_hash,DLM_HASH_PAGES ); +if (dlm->name)kfree (dlm->name);
@@ -1534,12 +1537,27 @@ static struct dlm_ctxt *dlm_alloc_ctxt(const char *domain,for (i =0 ; i <DLM_HASH_BUCKETS ; i++)INIT_HLIST_HEAD (dlm_lockres_hash (dlm, i)); + dlm->master_hash = (struct hlist_head **) +dlm_alloc_pagevec (DLM_HASH_PAGES ); +if (!dlm->master_hash) { +mlog_errno (-ENOMEM ); +dlm_free_pagevec ((void **)dlm->lockres_hash,DLM_HASH_PAGES ); +kfree (dlm->name); +kfree (dlm); + dlm =NULL ; + goto leave; + } + +for (i =0 ; i <DLM_HASH_BUCKETS ; i++) +INIT_HLIST_HEAD (dlm_master_hash (dlm, i)); +strcpy (dlm->name, domain); dlm->key = key; dlm->node_num =o2nm_this_node (); ret =dlm_create_debugfs_subroot (dlm);if (ret <0 ) { +dlm_free_pagevec ((void **)dlm->master_hash,DLM_HASH_PAGES );dlm_free_pagevec ((void **)dlm->lockres_hash,DLM_HASH_PAGES );kfree (dlm->name);kfree (dlm);
Signature · 1 line
1.5.6.3
PATCH 5/10 ocfs2/dlm: Activate dlm->master_hash for master list entries
In reply to Sunil MushranWith this patch, the mles are stored in a hash and not a simple list. This should improve the mle lookup time when the number of outstanding masteries is large.
--- fs/ocfs2/dlm/dlmcommon.h | 4 +- fs/ocfs2/dlm/dlmdebug.c | 24 ++++++++++++------- fs/ocfs2/dlm/dlmdomain.c | 1 - fs/ocfs2/dlm/dlmmaster.c | 58 ++++++++++++++++++++++++++++++++------------- 4 files changed, 58 insertions(+), 29 deletions(-)
fs/ocfs2/dlm/dlmcommon.h +2 -2
@@ -56,12 +56,13 @@ enum dlm_mle_type { };struct dlm_lock_name { +unsigned int hash;unsigned int len;unsigned char name[DLM_LOCKID_NAME_MAX ]; };struct dlm_master_list_entry { -struct list_head list; +struct hlist_node master_hash_node;struct list_head hb_events;struct dlm_ctxt *dlm;spinlock_t spinlock;
@@ -152,7 +153,6 @@ struct dlm_ctxtstruct dlm_recovery_ctxt reco;spinlock_t master_lock;struct hlist_head **master_hash; -struct list_head master_list;struct list_head mle_hb_events;/* these give a really vague idea of the system load */
fs/ocfs2/dlm/dlmdebug.c +15 -9
@@ -501,18 +501,25 @@ static struct file_operations debug_purgelist_fops = {static int debug_mle_print (struct dlm_ctxt *dlm,struct debug_buffer *db) {struct dlm_master_list_entry *mle; -int out =0 ; +struct hlist_head *bucket; +struct hlist_node *list; +int i, out =0 ;unsigned long total =0 ; out +=snprintf (db->buf + out, db->len - out,"Dumping MLEs for Domain: %s\n" , dlm->name);spin_lock (&dlm->master_lock); -list_for_each_entry (mle, &dlm->master_list, list) { - ++total; -if (db->len - out <200 ) -continue ; - out +=dump_mle (mle, db->buf + out, db->len - out); +for (i =0 ; i <DLM_HASH_BUCKETS ; i++) { + bucket =dlm_master_hash (dlm, i); +hlist_for_each (list, bucket) { + mle = hlist_entry(list, struct dlm_master_list_entry, + master_hash_node); + ++total; +if (db->len - out <200 ) +continue ; + out +=dump_mle (mle, db->buf + out, db->len - out); + } }spin_unlock (&dlm->master_lock);
@@ -813,12 +820,11 @@ static int debug_state_print(struct dlm_ctxt *dlm, struct debug_buffer *db)/* Lists: Dirty=Empty Purge=InUse PendingASTs=Empty ... */ out +=snprintf (db->buf + out, db->len - out,"Lists: Dirty=%s Purge=%s PendingASTs=%s " -"PendingBASTs=%s Master=%s\n" , +"PendingBASTs=%s\n" , (list_empty (&dlm->dirty_list) ?"Empty" :"InUse" ), (list_empty (&dlm->purge_list) ?"Empty" :"InUse" ), (list_empty (&dlm->pending_asts) ?"Empty" :"InUse" ), - (list_empty (&dlm->pending_basts) ?"Empty" :"InUse" ), - (list_empty (&dlm->master_list) ?"Empty" :"InUse" )); + (list_empty (&dlm->pending_basts) ?"Empty" :"InUse" ));/* Purge Count: xxx Refs: xxx */ out += snprintf(db->buf + out, db->len - out,
fs/ocfs2/dlm/dlmdomain.c +0 -1
@@ -1597,7 +1597,6 @@ static struct dlm_ctxt *dlm_alloc_ctxt(const char *domain,init_waitqueue_head (&dlm->reco.event);init_waitqueue_head (&dlm->ast_wq);init_waitqueue_head (&dlm->migration_wq); -INIT_LIST_HEAD (&dlm->master_list);INIT_LIST_HEAD (&dlm->mle_hb_events); dlm->joining_node =DLM_LOCK_RES_OWNER_UNKNOWN ;
fs/ocfs2/dlm/dlmmaster.c +41 -17
146 lines not on this page · raw message
Signature · 1 line
1.5.6.3
PATCH 6/10 ocfs2/dlm: Indent dlm_cleanup_master_list()
In reply to Sunil MushranThe previous patch explicitly did not indent dlm_cleanup_master_list() so as to make the patch readable. This patch properly indents the function.
--- fs/ocfs2/dlm/dlmmaster.c | 106 ++++++++++++++++++++++----------------------- 1 files changed, 52 insertions(+), 54 deletions(-)
fs/ocfs2/dlm/dlmmaster.c +52 -54
@@ -3320,66 +3320,64 @@ top: mle = hlist_entry(list, struct dlm_master_list_entry, master_hash_node); -BUG_ON (mle->type !=DLM_MLE_BLOCK && - mle->type !=DLM_MLE_MASTER && - mle->type !=DLM_MLE_MIGRATION ); - -/* MASTER mles are initiated locally. the waiting -* process will notice the node map change -* shortly. let that happen as normal. */ -if (mle->type ==DLM_MLE_MASTER ) -continue ; - +BUG_ON (mle->type !=DLM_MLE_BLOCK && + mle->type !=DLM_MLE_MASTER && + mle->type !=DLM_MLE_MIGRATION ); + +/* MASTER mles are initiated locally. The waiting +* process will notice the node map change shortly. +* Let that happen as normal. */ +if (mle->type ==DLM_MLE_MASTER ) +continue ; + +/* BLOCK mles are initiated by other nodes. Need to +* clean up if the dead node would have been the +* master. */ +if (mle->type ==DLM_MLE_BLOCK ) { +dlm_clean_block_mle (dlm, mle, dead_node); +continue ; + } -/* BLOCK mles are initiated by other nodes. -* need to clean up if the dead node would have -* been the master. */ -if (mle->type ==DLM_MLE_BLOCK ) { -dlm_clean_block_mle (dlm, mle, dead_node); -continue ; +/* Everything else is a MIGRATION mle */ + +/* The rule for MIGRATION mles is that the master +* becomes UNKNOWN if *either* the original or the new +* master dies. All UNKNOWN lockres' are sent to +* whichever node becomes the recovery master. The new +* master is responsible for determining if there is +* still a master for this lockres, or if he needs to +* take over mastery. Either way, this node should +* expect another message to resolve this. */ + +if (mle->master != dead_node && + mle->new_master != dead_node) +continue ; + +/* If we have reached this point, this mle needs to be +* removed from the list and freed. */ +dlm_clean_migration_mle (dlm, mle); + +mlog (0 ,"%s: node %u died during migration from " +"%u to %u!\n" , dlm->name, dead_node, mle->master, + mle->new_master); + +/* If we find a lockres associated with the mle, we've +* hit this rare case that messes up our lock ordering. +* If so, we need to drop the master lock so that we can +* take the lockres lock, meaning that we will have to +* restart from the head of list. */ + res =dlm_reset_mleres_owner (dlm, mle); +if (res) +/* restart */ + goto top; + +/* This may be the last reference */ +__dlm_put_mle (mle); } - -/* everything else is a MIGRATION mle */ - -/* the rule for MIGRATION mles is that the master -* becomes UNKNOWN if *either* the original or -* the new master dies. all UNKNOWN lockreses -* are sent to whichever node becomes the recovery -* master. the new master is responsible for -* determining if there is still a master for -* this lockres, or if he needs to take over -* mastery. either way, this node should expect -* another message to resolve this. */ -if (mle->master != dead_node && - mle->new_master != dead_node) -continue ; - -/* if we have reached this point, this mle needs to -* be removed from the list and freed. */ -dlm_clean_migration_mle (dlm, mle); - -mlog (0 ,"%s: node %u died during migration from " -"%u to %u!\n" , dlm->name, dead_node, - mle->master, mle->new_master); - -/* If we find a lockres associated with the mle, we've -* hit this rare case that messes up our lock ordering. -* If so, we need to drop the master lock so that we can -* take the lockres lock, meaning that we will have to -* restart from the head of list. */ - res =dlm_reset_mleres_owner (dlm, mle); -if (res) -/* restart */ - goto top; - -/* this may be the last reference */ -__dlm_put_mle (mle); - } }spin_unlock (&dlm->master_lock); } -int dlm_finish_migration (struct dlm_ctxt *dlm,struct dlm_lock_resource *res,u8 old_master) {
Signature · 1 line
1.5.6.3
PATCH 7/10 ocfs2/dlm: Track number of mles
In reply to Sunil MushranThe lifetime of a mle is limited to the duration of the lockres mastery process. While typically this lifetime is fairly short, we have noticed the number of mles explode under certain circumstances. This patch tracks the number of each different types of mles and should help us determine how best to speed up the mastery process.
--- fs/ocfs2/dlm/dlmcommon.h | 5 ++++- fs/ocfs2/dlm/dlmdomain.c | 5 +++++ fs/ocfs2/dlm/dlmmaster.c | 5 +++++ 3 files changed, 14 insertions(+), 1 deletions(-)
fs/ocfs2/dlm/dlmcommon.h +4 -1
@@ -52,7 +52,8 @@enum dlm_mle_type {DLM_MLE_BLOCK ,DLM_MLE_MASTER , -DLM_MLE_MIGRATION +DLM_MLE_MIGRATION , +DLM_MLE_NUM_TYPES };struct dlm_lock_name {
@@ -156,6 +157,8 @@ struct dlm_ctxtstruct list_head mle_hb_events;/* these give a really vague idea of the system load */ +atomic_t mle_tot_count[DLM_MLE_NUM_TYPES ]; +atomic_t mle_cur_count[DLM_MLE_NUM_TYPES ];atomic_t local_resources;atomic_t remote_resources;atomic_t unknown_resources;
fs/ocfs2/dlm/dlmdomain.c +5 -0
@@ -1608,6 +1608,11 @@ static struct dlm_ctxt *dlm_alloc_ctxt(const char *domain,atomic_set (&dlm->remote_resources,0 );atomic_set (&dlm->unknown_resources,0 ); +for (i =0 ; i <DLM_MLE_NUM_TYPES ; ++i) { +atomic_set (&dlm->mle_tot_count[i],0 ); +atomic_set (&dlm->mle_cur_count[i],0 ); + } +spin_lock_init (&dlm->work_lock);INIT_LIST_HEAD (&dlm->work_list);INIT_WORK (&dlm->dispatched_work, dlm_dispatch_work);
fs/ocfs2/dlm/dlmmaster.c +5 -0
@@ -325,6 +325,9 @@ static void dlm_init_mle(struct dlm_master_list_entry *mle, mle->u.mlename.hash =dlm_lockid_hash (name, namelen); } +atomic_inc (&dlm->mle_tot_count[mle->type]); +atomic_inc (&dlm->mle_cur_count[mle->type]); +/* copy off the node_map and register hb callbacks on our copy */ memcpy (mle->node_map, dlm->domain_map,sizeof (mle->node_map));memcpy (mle->vote_map, dlm->domain_map,sizeof (mle->vote_map));
@@ -467,6 +470,8 @@ static void dlm_mle_release(struct kref *kref)/* detach the mle from the domain node up/down events */ __dlm_mle_detach_hb_events (dlm, mle); +atomic_dec (&dlm->mle_cur_count[mle->type]); +/* NOTE: kfree under spinlock here. * if this is bad, we can move this to a freelist. */ kmem_cache_free (dlm_mle_cache, mle);
Signature · 1 line
1.5.6.3
PATCH 8/10 ocfs2/dlm: Improve lockres counts
In reply to Sunil MushranThis patch replaces the lockres counts that tracked the number number of locally and remotely mastered lockres' with a current and total count. The total count is the number of lockres' that have been created since the dlm domain was created.
The number of locally and remotely mastered counts can be computed using the locking_state output.
--- fs/ocfs2/dlm/dlmcommon.h | 5 ++--- fs/ocfs2/dlm/dlmdebug.c | 12 ------------ fs/ocfs2/dlm/dlmdomain.c | 5 ++--- fs/ocfs2/dlm/dlmmaster.c | 27 +++++++-------------------- 4 files changed, 11 insertions(+), 38 deletions(-)
fs/ocfs2/dlm/dlmcommon.h +2 -3
@@ -159,9 +159,8 @@ struct dlm_ctxt/* these give a really vague idea of the system load */ atomic_t mle_tot_count[DLM_MLE_NUM_TYPES ];atomic_t mle_cur_count[DLM_MLE_NUM_TYPES ]; -atomic_t local_resources; -atomic_t remote_resources; -atomic_t unknown_resources; +atomic_t res_tot_count; +atomic_t res_cur_count;struct dlm_debug_ctxt *dlm_debug_ctxt;struct dentry *dlm_debugfs_subroot;
fs/ocfs2/dlm/dlmdebug.c +0 -12
@@ -763,12 +763,6 @@ static int debug_state_print(struct dlm_ctxt *dlm, struct debug_buffer *db)int out =0 ;struct dlm_reco_node_data *node;char *state; -int lres, rres, ures, tres; - - lres =atomic_read (&dlm->local_resources); - rres =atomic_read (&dlm->remote_resources); - ures =atomic_read (&dlm->unknown_resources); - tres = lres + rres + ures;spin_lock (&dlm->spinlock);
@@ -811,12 +805,6 @@ static int debug_state_print(struct dlm_ctxt *dlm, struct debug_buffer *db) db->buf + out, db->len - out); out +=snprintf (db->buf + out, db->len - out,"\n" ); -/* Mastered Resources Total: xxx Locally: xxx Remotely: ... */ - out +=snprintf (db->buf + out, db->len - out, -"Mastered Resources Total: %d Locally: %d " -"Remotely: %d Unknown: %d\n" , - tres, lres, rres, ures); -/* Lists: Dirty=Empty Purge=InUse PendingASTs=Empty ... */ out +=snprintf (db->buf + out, db->len - out, "Lists: Dirty=%s Purge=%s PendingASTs=%s "
fs/ocfs2/dlm/dlmdomain.c +2 -3
@@ -1604,10 +1604,9 @@ static struct dlm_ctxt *dlm_alloc_ctxt(const char *domain, dlm->reco.new_master =O2NM_INVALID_NODE_NUM ; dlm->reco.dead_node =O2NM_INVALID_NODE_NUM ; -atomic_set (&dlm->local_resources,0 ); -atomic_set (&dlm->remote_resources,0 ); -atomic_set (&dlm->unknown_resources,0 ); +atomic_set (&dlm->res_tot_count,0 ); +atomic_set (&dlm->res_cur_count,0 );for (i =0 ; i <DLM_MLE_NUM_TYPES ; ++i) {atomic_set (&dlm->mle_tot_count[i],0 );atomic_set (&dlm->mle_cur_count[i],0 );
fs/ocfs2/dlm/dlmmaster.c +7 -20
@@ -517,15 +517,6 @@ static void dlm_set_lockres_owner(struct dlm_ctxt *dlm, {assert_spin_locked (&res->spinlock); -mlog_entry ("%.*s, %u\n" , res->lockname.len, res->lockname.name, owner); - -if (owner == dlm->node_num) -atomic_inc (&dlm->local_resources); -else if (owner ==DLM_LOCK_RES_OWNER_UNKNOWN ) -atomic_inc (&dlm->unknown_resources); -else -atomic_inc (&dlm->remote_resources); - res->owner = owner; }
@@ -534,17 +525,8 @@ void dlm_change_lockres_owner(struct dlm_ctxt *dlm, {assert_spin_locked (&res->spinlock); -if (owner == res->owner) -return ; - -if (res->owner == dlm->node_num) -atomic_dec (&dlm->local_resources); -else if (res->owner ==DLM_LOCK_RES_OWNER_UNKNOWN ) -atomic_dec (&dlm->unknown_resources); -else -atomic_dec (&dlm->remote_resources); - -dlm_set_lockres_owner (dlm, res, owner); +if (owner != res->owner) +dlm_set_lockres_owner (dlm, res, owner); }
@@ -573,6 +555,8 @@ static void dlm_lockres_release(struct kref *kref) }spin_unlock (&dlm->track_lock); +atomic_dec (&dlm->res_cur_count); +dlm_put (dlm); if (!hlist_unhashed(&res->hash_node) ||
@@ -653,6 +637,9 @@ static void dlm_init_lockres(struct dlm_ctxt *dlm,kref_init (&res->refs); +atomic_inc (&dlm->res_tot_count); +atomic_inc (&dlm->res_cur_count); +/* just for consistency */ spin_lock (&res->spinlock);dlm_set_lockres_owner (dlm, res,DLM_LOCK_RES_OWNER_UNKNOWN );
Signature · 1 line
1.5.6.3
PATCH 9/10 ocfs2/dlm: dlm_set_lockres_owner() and dlm_change_lockres_owner() inlined
In reply to Sunil MushranThis patch inlines dlm_set_lockres_owner() and dlm_change_lockres_owner().
--- fs/ocfs2/dlm/dlmcommon.h | 21 ++++++++++++++++++--- fs/ocfs2/dlm/dlmmaster.c | 19 ------------------- 2 files changed, 18 insertions(+), 22 deletions(-)
fs/ocfs2/dlm/dlmcommon.h +18 -3
@@ -858,9 +858,7 @@ struct dlm_lock_resource * dlm_lookup_lockres(struct dlm_ctxt *dlm, unsigned int len);int dlm_is_host_down (int errno); -void dlm_change_lockres_owner (struct dlm_ctxt *dlm, -struct dlm_lock_resource *res, -u8 owner); + struct dlm_lock_resource * dlm_get_lock_resource(struct dlm_ctxt *dlm, const char *lockid, int namelen,
@@ -1123,6 +1121,23 @@ static inline int dlm_node_iter_next(struct dlm_node_iter *iter) return bit; } +static inline void dlm_set_lockres_owner (struct dlm_ctxt *dlm, +struct dlm_lock_resource *res, +u8 owner) +{ +assert_spin_locked (&res->spinlock); + + res->owner = owner; +} +static inline void dlm_change_lockres_owner (struct dlm_ctxt *dlm, +struct dlm_lock_resource *res, +u8 owner) +{ +assert_spin_locked (&res->spinlock); + +if (owner != res->owner) +dlm_set_lockres_owner (dlm, res, owner); +}#endif /* DLMCOMMON_H */
fs/ocfs2/dlm/dlmmaster.c +0 -19
@@ -511,25 +511,6 @@ void dlm_destroy_master_caches(void)kmem_cache_destroy (dlm_lockres_cache); } -static void dlm_set_lockres_owner (struct dlm_ctxt *dlm, -struct dlm_lock_resource *res, -u8 owner) -{ -assert_spin_locked (&res->spinlock); - - res->owner = owner; -} - -void dlm_change_lockres_owner (struct dlm_ctxt *dlm, -struct dlm_lock_resource *res,u8 owner) -{ -assert_spin_locked (&res->spinlock); - -if (owner != res->owner) -dlm_set_lockres_owner (dlm, res, owner); -} - -static void dlm_lockres_release (struct kref *kref) {struct dlm_lock_resource *res;
Signature · 1 line
1.5.6.3
PATCH 10/10 ocfs2/dlm: Show the number of lockres/mles in dlm_state
In reply to Sunil MushranThis patch shows the number of lockres' and mles in the debugfs file, dlm_state.
--- fs/ocfs2/dlm/dlmdebug.c | 36 ++++++++++++++++++++++++++++++++++++ 1 files changed, 36 insertions(+), 0 deletions(-)
fs/ocfs2/dlm/dlmdebug.c +36 -0
@@ -763,6 +763,8 @@ static int debug_state_print(struct dlm_ctxt *dlm, struct debug_buffer *db)int out =0 ;struct dlm_reco_node_data *node;char *state; +int cur_mles =0 , tot_mles =0 ; +int i;spin_lock (&dlm->spinlock);
@@ -805,6 +807,40 @@ static int debug_state_print(struct dlm_ctxt *dlm, struct debug_buffer *db) db->buf + out, db->len - out); out +=snprintf (db->buf + out, db->len - out,"\n" ); +/* Lock Resources: xxx (xxx) */ + out +=snprintf (db->buf + out, db->len - out, +"Lock Resources: %d (%d)\n" , +atomic_read (&dlm->res_cur_count), +atomic_read (&dlm->res_tot_count)); + +for (i =0 ; i <DLM_MLE_NUM_TYPES ; ++i) + tot_mles +=atomic_read (&dlm->mle_tot_count[i]); + +for (i =0 ; i <DLM_MLE_NUM_TYPES ; ++i) + cur_mles +=atomic_read (&dlm->mle_cur_count[i]); + +/* MLEs: xxx (xxx) */ + out +=snprintf (db->buf + out, db->len - out, +"MLEs: %d (%d)\n" , cur_mles, tot_mles); + +/* Blocking: xxx (xxx) */ + out +=snprintf (db->buf + out, db->len - out, +" Blocking: %d (%d)\n" , +atomic_read (&dlm->mle_cur_count[DLM_MLE_BLOCK ]), +atomic_read (&dlm->mle_tot_count[DLM_MLE_BLOCK ])); + +/* Mastery: xxx (xxx) */ + out +=snprintf (db->buf + out, db->len - out, +" Mastery: %d (%d)\n" , +atomic_read (&dlm->mle_cur_count[DLM_MLE_MASTER ]), +atomic_read (&dlm->mle_tot_count[DLM_MLE_MASTER ])); + +/* Migration: xxx (xxx) */ + out +=snprintf (db->buf + out, db->len - out, +" Migration: %d (%d)\n" , +atomic_read (&dlm->mle_cur_count[DLM_MLE_MIGRATION ]), +atomic_read (&dlm->mle_tot_count[DLM_MLE_MIGRATION ])); +/* Lists: Dirty=Empty Purge=InUse PendingASTs=Empty ... */ out +=snprintf (db->buf + out, db->len - out, "Lists: Dirty=%s Purge=%s PendingASTs=%s "
Signature · 1 line
1.5.6.3