Skip to contentSkip to conversation

dlm: refcount fixes and improvements

PATCH New Alexander Aring 1 participant 3 messages Jan 4, 2024 Distributed lock manager (DLM)

lore
Share & tools
lore

Alexander Aring

author maintainer
#0 lore raw

cover dlm: refcount fixes and improvements

Hi,

those two patches contains a patch to fix a potential refcount issue in DLM and improve the freeing of lkbs at lockspace release time. Now I can use /proc/slabinfo to see if all lkbs, rsbs got freed up, before it was not possible to do that because we was doing a "forced" free of all datastructures and didn't respect the refcount programming logic.

I need to send v2 for the softirq patches, I will do that soon.

- Alex

Alexander Aring (2):
dlm: fix off-by-one waiters refcount handling
dlm: put lkbs instead of force free

 fs/dlm/lock.c      | 12 +++++++-----
 fs/dlm/lock.h      |  1 +
 fs/dlm/lockspace.c | 31 +++++++++++++++++++++----------
 3 files changed, 29 insertions(+), 15 deletions(-)
Signature · 1 line
2.39.3
author maintainer
#1 lore raw

PATCH 1/2 dlm: fix off-by-one waiters refcount handling

There was a wrong conversion to atomic counters in commit 75a7d60134ce ("fs: dlm: handle lkb wait count as atomic_t"), when atomic_dec_and_test() returns true it will decrement at first and then return true if it hits zero. This means we will mis a unhold_lkb() for the last iteration. This patch fixes this issue and if the last reference is taken we will remove the lkb from the waiters list as this is how it's supposed to work.

Fixes 75a7d60134ce ("fs: dlm: handle lkb wait count as atomic_t")
---
 fs/dlm/lock.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
fs/dlm/lock.c +6 -4
index 652c51fbbf76..c30e9f8d017e 100644
@@ -5070,11 +5070,13 @@ int dlm_recover_waiters_post(struct dlm_ls *ls)
 		/* drop all wait_count references we still
 		 * hold a reference for this iteration.
 		 */
-		while (!atomic_dec_and_test(&lkb->lkb_wait_count))
-			unhold_lkb(lkb);
-
 		mutex_lock(&ls->ls_waiters_mutex);
-		list_del_init(&lkb->lkb_wait_reply);
+		while (atomic_read(&lkb->lkb_wait_count)) {
+			if (atomic_dec_and_test(&lkb->lkb_wait_count))
+				list_del_init(&lkb->lkb_wait_reply);
+
+			unhold_lkb(lkb);
+		}
 		mutex_unlock(&ls->ls_waiters_mutex);
 
 		if (oc || ou) {
Signature · 1 line
2.39.3
author maintainer
#2 lore raw

PATCH 2/2 dlm: put lkbs instead of force free

In reply to Alexander Aring

This patch converts a force free of the lkb idr and switch to use the lkbs put functionality. If there are still references hold due the lkb programming logic and its state it will be drop before. Instead of force freeing the lkbs of the idr using the refcounters makes sure we using the reference counters correctly. If we do that, then no rsb should be left on the lockspace keep hash bucket which is an additonally check added to this patch. All rsbs on the toss list should have a reference counter of 1.

---
 fs/dlm/lock.c      |  2 +-
 fs/dlm/lock.h      |  1 +
 fs/dlm/lockspace.c | 31 +++++++++++++++++++++----------
 3 files changed, 23 insertions(+), 11 deletions(-)
fs/dlm/lock.c +1 -1
index c30e9f8d017e..f77f479e53b6 100644
@@ -1368,7 +1368,7 @@ static void add_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb, int status)
 	}
 }
 
-static void del_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb)
+void del_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb)
 {
 	lkb->lkb_status = 0;
 	list_del(&lkb->lkb_statequeue);
fs/dlm/lock.h +1 -0
index b54e2cbbe6e2..853c3d3dc49d 100644
@@ -60,6 +60,7 @@ int dlm_debug_add_lkb(struct dlm_ls *ls, uint32_t lkb_id, char *name, int len,
 		      int lkb_nodeid, unsigned int lkb_flags, int lkb_status);
 int dlm_debug_add_lkb_to_waiters(struct dlm_ls *ls, uint32_t lkb_id,
 				 int mstype, int to_nodeid);
+void del_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb);
 
 static inline int is_master(struct dlm_rsb *r)
 {
fs/dlm/lockspace.c +21 -10
index 0455dddb0797..c7ab7358422b 100644
@@ -737,14 +737,28 @@ static int lkb_idr_is_any(int id, void *p, void *data)
 	return 1;
 }
 
-static int lkb_idr_free(int id, void *p, void *data)
+/*
+ * No locking required, lockspace usage should be synchronized
+ * to have any activity anymore.
+ */
+static int lkb_idr_put(int id, void *p, void *data)
 {
 	struct dlm_lkb *lkb = p;
 
-	if (lkb->lkb_lvbptr && test_bit(DLM_IFL_MSTCPY_BIT, &lkb->lkb_iflags))
-		dlm_free_lvb(lkb->lkb_lvbptr);
+	if (lkb->lkb_status)
+		del_lkb(lkb->lkb_resource, lkb);
 
-	dlm_free_lkb(lkb);
+	/* drop all wait_count references we still
+	 * hold a reference for this iteration.
+	 */
+	while (atomic_read(&lkb->lkb_wait_count)) {
+		if (atomic_dec_and_test(&lkb->lkb_wait_count))
+			list_del_init(&lkb->lkb_wait_reply);
+
+		WARN_ON_ONCE(dlm_put_lkb(lkb));
+	}
+
+	WARN_ON_ONCE(!dlm_put_lkb(lkb));
 	return 0;
 }
 
@@ -826,7 +840,7 @@ static int release_lockspace(struct dlm_ls *ls, int force)
 	 * Free all lkb's in idr
 	 */
 
-	idr_for_each(&ls->ls_lkbidr, lkb_idr_free, ls);
+	idr_for_each(&ls->ls_lkbidr, lkb_idr_put, ls);
 	idr_destroy(&ls->ls_lkbidr);
 
 	/*
@@ -834,15 +848,12 @@ static int release_lockspace(struct dlm_ls *ls, int force)
 	 */
 
 	for (i = 0; i < ls->ls_rsbtbl_size; i++) {
-		while ((n = rb_first(&ls->ls_rsbtbl[i].keep))) {
-			rsb = rb_entry(n, struct dlm_rsb, res_hashnode);
-			rb_erase(n, &ls->ls_rsbtbl[i].keep);
-			dlm_free_rsb(rsb);
-		}
+		WARN_ON_ONCE(!RB_EMPTY_ROOT(&ls->ls_rsbtbl[i].keep));
 
 		while ((n = rb_first(&ls->ls_rsbtbl[i].toss))) {
 			rsb = rb_entry(n, struct dlm_rsb, res_hashnode);
 			rb_erase(n, &ls->ls_rsbtbl[i].toss);
+			WARN_ON_ONCE(kref_read(&rsb->res_ref) != 1);
 			dlm_free_rsb(rsb);
 		}
 	}
Signature · 1 line
2.39.3