Skip to contentSkip to conversation

gfs2 cleanups and fixes

PATCH In review Andreas Gruenbacher 2 participants 19 messages Apr 9, 2024 – Apr 17, 2024

Subsystems (2)
lore
Share & tools
lore

Andreas Gruenbacher

author maintainer
#0 lore raw

cover gfs2 cleanups and fixes

The following gfs2 cleanups and fixes have accumulated recently. I've queued them up for the next merge window; see the for-next branch at:

https://git.kernel.org/pub/scm/linux/kernel/git/gfs2/linux-gfs2.git/

Thanks,
Andreas

Andreas Gruenbacher (15):
gfs2: Get rid of newlines in log messages
gfs2: Remove unnecessary gfs2_meta_check_ii argument
gfs2: Follow-up to flag rename in sysfs status file
gfs2: Use [NO_]CREATE consistently for gfs2_glock_get
gfs2: Don't forget to complete delayed withdraw
gfs2: Fix NULL pointer dereference in gfs2_log_flush
gfs2: Get rid of gfs2_glock_queue_put in signal_our_withdraw
gfs2: Replace gfs2_glock_queue_put with gfs2_glock_put_async
gfs2: Don't set GLF_LOCK in gfs2_dispose_glock_lru
gfs2: Get rid of unnecessary test_and_set_bit
gfs2: Fix "ignore unlock failures after withdraw" Revert "gfs2: fix glock shrinker ref issues"
gfs2: Fix "Make glock lru list scanning safer"
gfs2: Get rid of demote_ok checks
gfs2: Fix lru_count accounting

 Documentation/filesystems/gfs2-glocks.rst |   3 -
 fs/gfs2/bmap.c                            |   2 +-
 fs/gfs2/glock.c                           | 101 ++++++++++------------
 fs/gfs2/glock.h                           |   2 +-
 fs/gfs2/glops.c                           |  21 +----
 fs/gfs2/incore.h                          |   1 -
 fs/gfs2/log.c                             |   5 +-
 fs/gfs2/rgrp.c                            |  10 +--
 fs/gfs2/super.c                           |   8 +-
 fs/gfs2/sys.c                             |   4 +-
 fs/gfs2/util.c                            |  57 ++++++------
 fs/gfs2/util.h                            |   4 +-
 12 files changed, 96 insertions(+), 122 deletions(-)
Signature · 1 line
2.44.0
author maintainer
#1 lore raw

PATCH 1/15 gfs2: Get rid of newlines in log messages

Get rid of attempts to create multi-line syslog entries; this only makes the messages harder to read.

---
 fs/gfs2/bmap.c |  2 +-
 fs/gfs2/rgrp.c | 10 +++++-----
 fs/gfs2/util.c | 50 ++++++++++++++++++++++++++------------------------
 3 files changed, 32 insertions(+), 30 deletions(-)
fs/gfs2/bmap.c +1 -1
index 643175498d1c..b8d64f2259d6 100644
@@ -1827,7 +1827,7 @@ static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length)
 			gfs2_assert_withdraw(sdp, bh);
 			if (gfs2_assert_withdraw(sdp,
 						 prev_bnr != bh->b_blocknr)) {
-				fs_emerg(sdp, "inode %llu, block:%llu, i_h:%u,"
+				fs_emerg(sdp, "inode %llu, block:%llu, i_h:%u, "
 					 "s_h:%u, mp_h:%u\n",
 				       (unsigned long long)ip->i_no_addr,
 				       prev_bnr, ip->i_height, strip_h, mp_h);
fs/gfs2/rgrp.c +5 -5
index 26d6c1eea559..29c772816765 100644
@@ -814,11 +814,11 @@ static int compute_bitstructs(struct gfs2_rgrpd *rgd)
 	bi = rgd->rd_bits + (length - 1);
 	if ((bi->bi_start + bi->bi_bytes) * GFS2_NBBY != rgd->rd_data) {
 		gfs2_lm(sdp,
-			"ri_addr = %llu\n"
-			"ri_length = %u\n"
-			"ri_data0 = %llu\n"
-			"ri_data = %u\n"
-			"ri_bitbytes = %u\n"
+			"ri_addr=%llu "
+			"ri_length=%u "
+			"ri_data0=%llu "
+			"ri_data=%u "
+			"ri_bitbytes=%u "
 			"start=%u len=%u offset=%u\n",
 			(unsigned long long)rgd->rd_addr,
 			rgd->rd_length,
fs/gfs2/util.c +26 -24

111 lines not on this page · raw message

Signature · 1 line
2.44.0
author maintainer
#2 lore raw

PATCH 2/15 gfs2: Remove unnecessary gfs2_meta_check_ii argument

In reply to Andreas Gruenbacher

The type argument of gfs2_meta_check_ii() is always set to "magic number", so remove that argument and hardcode the string in gfs2_meta_check_ii(). Change the string to "bad magic number" to emphasize that the problem is the incorrect magic number.

---
 fs/gfs2/util.c | 6 +++---
 fs/gfs2/util.h | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)
fs/gfs2/util.c +3 -3
index fd65a2e34102..963d77c4ca21 100644
@@ -488,16 +488,16 @@ void gfs2_consist_rgrpd_i(struct gfs2_rgrpd *rgd,
  */
 
 int gfs2_meta_check_ii(struct gfs2_sbd *sdp, struct buffer_head *bh,
-		       const char *type, const char *function, char *file,
+		       const char *function, char *file,
 		       unsigned int line)
 {
 	int me;
 
 	gfs2_lm(sdp,
 		"fatal: invalid metadata block - "
-		"bh = %llu (%s), "
+		"bh = %llu (bad magic number), "
 		"function = %s, file = %s, line = %u\n",
-		(unsigned long long)bh->b_blocknr, type,
+		(unsigned long long)bh->b_blocknr,
 		function, file, line);
 	me = gfs2_withdraw(sdp);
 	return (me) ? -1 : -2;
fs/gfs2/util.h +2 -2
index ba071998461f..d8d62bf0424d 100644
@@ -92,7 +92,7 @@ gfs2_consist_rgrpd_i((rgd), __func__, __FILE__, __LINE__)
 
 
 int gfs2_meta_check_ii(struct gfs2_sbd *sdp, struct buffer_head *bh,
-		       const char *type, const char *function,
+		       const char *function,
 		       char *file, unsigned int line);
 
 static inline int gfs2_meta_check(struct gfs2_sbd *sdp,
@@ -123,7 +123,7 @@ static inline int gfs2_metatype_check_i(struct gfs2_sbd *sdp,
 	u32 magic = be32_to_cpu(mh->mh_magic);
 	u16 t = be32_to_cpu(mh->mh_type);
 	if (unlikely(magic != GFS2_MAGIC))
-		return gfs2_meta_check_ii(sdp, bh, "magic number", function,
+		return gfs2_meta_check_ii(sdp, bh, function,
 					  file, line);
         if (unlikely(t != type))
 		return gfs2_metatype_check_ii(sdp, bh, type, t, function,
Signature · 1 line
2.44.0
author maintainer
#3 lore raw

PATCH 3/15 gfs2: Follow-up to flag rename in sysfs status file

In reply to Andreas Gruenbacher

Follow up to commit e7beb8b6de1a ("gfs2: Rename SDF_DEACTIVATING to SDF_KILL") and change the description of the SDF_KILL flag from "Deactivating" to "Killing".

---
 fs/gfs2/sys.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
fs/gfs2/sys.c +1 -1
index 250f340cb44d..32d3484270f3 100644
@@ -88,7 +88,7 @@ static ssize_t status_show(struct gfs2_sbd *sdp, char *buf)
 		     "Withdraw In Prog:         %d\n"
 		     "Remote Withdraw:          %d\n"
 		     "Withdraw Recovery:        %d\n"
-		     "Deactivating:             %d\n"
+		     "Killing:                  %d\n"
 		     "sd_log_error:             %d\n"
 		     "sd_log_flush_lock:        %d\n"
 		     "sd_log_num_revoke:        %u\n"
Signature · 1 line
2.44.0
author maintainer
#4 lore raw

PATCH 4/15 gfs2: Use [NO_]CREATE consistently for gfs2_glock_get

In reply to Andreas Gruenbacher

When calling gfs2_glock_get(), consistently pass the CREATE or NO_CREATE flag for the create argument. This is the only place that passes 0 instead.

---
 fs/gfs2/sys.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
fs/gfs2/sys.c +1 -1
index 32d3484270f3..ecc699f8d9fc 100644
@@ -336,7 +336,7 @@ static ssize_t demote_rq_store(struct gfs2_sbd *sdp, const char *buf, size_t len
 		return -EINVAL;
 	if (!test_and_set_bit(SDF_DEMOTE, &sdp->sd_flags))
 		fs_info(sdp, "demote interface used\n");
-	rv = gfs2_glock_get(sdp, glnum, glops, 0, &gl);
+	rv = gfs2_glock_get(sdp, glnum, glops, NO_CREATE, &gl);
 	if (rv)
 		return rv;
 	gfs2_glock_cb(gl, glmode);
Signature · 1 line
2.44.0
author maintainer
#5 lore raw

PATCH 5/15 gfs2: Don't forget to complete delayed withdraw

In reply to Andreas Gruenbacher

Commit fffe9bee14b0 ("gfs2: Delay withdraw from atomic context") switched from gfs2_withdraw() to gfs2_withdraw_delayed() in gfs2_ail_error(), but failed to then check if a delayed withdraw had occurred. Fix that by adding the missing check in __gfs2_ail_flush(), where the spin locks are already dropped and a withdraw is possible.

Fixes fffe9bee14b0 ("gfs2: Delay withdraw from atomic context")
---
 fs/gfs2/glops.c | 3 +++
 1 file changed, 3 insertions(+)
fs/gfs2/glops.c +3 -0
index ae8d4731907d..68677fb69a73 100644
@@ -82,6 +82,9 @@ static void __gfs2_ail_flush(struct gfs2_glock *gl, bool fsync,
 	GLOCK_BUG_ON(gl, !fsync && atomic_read(&gl->gl_ail_count));
 	spin_unlock(&sdp->sd_ail_lock);
 	gfs2_log_unlock(sdp);
+
+	if (gfs2_withdrawing(sdp))
+		gfs2_withdraw(sdp);
 }
 
 
Signature · 1 line
2.44.0
author maintainer
#6 lore raw

PATCH 6/15 gfs2: Fix NULL pointer dereference in gfs2_log_flush

In reply to Andreas Gruenbacher

In gfs2_jindex_free(), set sdp->sd_jdesc to NULL under the log flush lock to provide exclusion against gfs2_log_flush().

In gfs2_log_flush(), check if sdp->sd_jdesc is non-NULL before dereferencing it. Otherwise, we could run into a NULL pointer dereference when outstanding glock work races with an unmount (glock_work_func -> run_queue -> do_xmote -> inode_go_sync -> gfs2_log_flush).

---
 fs/gfs2/log.c   | 3 ++-
 fs/gfs2/super.c | 4 ++++
 2 files changed, 6 insertions(+), 1 deletion(-)
fs/gfs2/log.c +2 -1
index 8cddf955ebc0..a6dd68b458ce 100644
@@ -1108,7 +1108,8 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags)
 	lops_before_commit(sdp, tr);
 	if (gfs2_withdrawing_or_withdrawn(sdp))
 		goto out_withdraw;
-	gfs2_log_submit_bio(&sdp->sd_jdesc->jd_log_bio, REQ_OP_WRITE);
+	if (sdp->sd_jdesc)
+		gfs2_log_submit_bio(&sdp->sd_jdesc->jd_log_bio, REQ_OP_WRITE);
 	if (gfs2_withdrawing_or_withdrawn(sdp))
 		goto out_withdraw;
 
fs/gfs2/super.c +4 -0
index e5f79466340d..9de789b78bc5 100644
@@ -67,9 +67,13 @@ void gfs2_jindex_free(struct gfs2_sbd *sdp)
 	sdp->sd_journals = 0;
 	spin_unlock(&sdp->sd_jindex_spin);
 
+	down_write(&sdp->sd_log_flush_lock);
 	sdp->sd_jdesc = NULL;
+	up_write(&sdp->sd_log_flush_lock);
+
 	while (!list_empty(&list)) {
 		jd = list_first_entry(&list, struct gfs2_jdesc, jd_list);
+		BUG_ON(jd->jd_log_bio);
 		gfs2_free_journal_extents(jd);
 		list_del(&jd->jd_list);
 		iput(jd->jd_inode);
Signature · 1 line
2.44.0
author maintainer
#7 lore raw

PATCH 7/15 gfs2: Get rid of gfs2_glock_queue_put in signal_our_withdraw

In reply to Andreas Gruenbacher

In function signal_our_withdraw(), we are calling gfs2_glock_queue_put() in a context in which we are actually allowed to sleep, so replace that with a simple call to gfs2_glock_put().

---
 fs/gfs2/util.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
fs/gfs2/util.c +1 -1
index 963d77c4ca21..0535a0206512 100644
@@ -255,7 +255,7 @@ static void signal_our_withdraw(struct gfs2_sbd *sdp)
 		gfs2_glock_nq(&sdp->sd_live_gh);
 	}
 
-	gfs2_glock_queue_put(live_gl); /* drop extra reference we acquired */
+	gfs2_glock_put(live_gl); /* drop extra reference we acquired */
 	clear_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags);
 
 	/*
Signature · 1 line
2.44.0
author maintainer
#8 lore raw

PATCH 8/15 gfs2: Replace gfs2_glock_queue_put with gfs2_glock_put_async

In reply to Andreas Gruenbacher

Function gfs2_glock_queue_put() puts a glock reference by enqueuing glock work instead of putting the reference directly. This ensures that the operation won't sleep, but it is costly and really only necessary when putting the final glock reference. Replace it with a new gfs2_glock_put_async() function that only queues glock work when putting the last glock reference.

---
 fs/gfs2/glock.c | 27 +++++++++++++++++----------
 fs/gfs2/glock.h |  2 +-
 fs/gfs2/log.c   |  2 +-
 fs/gfs2/super.c |  4 ++--
 4 files changed, 21 insertions(+), 14 deletions(-)
fs/gfs2/glock.c +17 -10
index 34540f9d011c..ed90033b9c72 100644
@@ -285,14 +285,6 @@ static void __gfs2_glock_put(struct gfs2_glock *gl)
 	sdp->sd_lockstruct.ls_ops->lm_put_lock(gl);
 }
 
-/*
- * Cause the glock to be put in work queue context.
- */
-void gfs2_glock_queue_put(struct gfs2_glock *gl)
-{
-	gfs2_glock_queue_work(gl, 0);
-}
-
 /**
  * gfs2_glock_put() - Decrement reference count on glock
  * @gl: The glock to put
@@ -307,6 +299,22 @@ void gfs2_glock_put(struct gfs2_glock *gl)
 	__gfs2_glock_put(gl);
 }
 
+/*
+ * gfs2_glock_put_async - Decrement reference count without sleeping
+ * @gl: The glock to put
+ *
+ * Decrement the reference count on glock immediately unless it is the last
+ * reference.  Defer putting the last reference to work queue context.
+ */
+void gfs2_glock_put_async(struct gfs2_glock *gl)
+{
+	if (lockref_put_or_lock(&gl->gl_lockref))
+		return;
+
+	__gfs2_glock_queue_work(gl, 0);
+	spin_unlock(&gl->gl_lockref.lock);
+}
+
 /**
  * may_grant - check if it's ok to grant a new lock
  * @gl: The glock
@@ -2529,8 +2537,7 @@ static void gfs2_glock_iter_next(struct gfs2_glock_iter *gi, loff_t n)
 	if (gl) {
 		if (n == 0)
 			return;
-		if (!lockref_put_not_zero(&gl->gl_lockref))
-			gfs2_glock_queue_put(gl);
+		gfs2_glock_put_async(gl);
 	}
 	for (;;) {
 		gl = rhashtable_walk_next(&gi->hti);
fs/gfs2/glock.h +1 -1
index 0114f3e0ebe0..2c697674a86f 100644
@@ -172,7 +172,7 @@ int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
 		   int create, struct gfs2_glock **glp);
 struct gfs2_glock *gfs2_glock_hold(struct gfs2_glock *gl);
 void gfs2_glock_put(struct gfs2_glock *gl);
-void gfs2_glock_queue_put(struct gfs2_glock *gl);
+void gfs2_glock_put_async(struct gfs2_glock *gl);
 
 void __gfs2_holder_init(struct gfs2_glock *gl, unsigned int state,
 		        u16 flags, struct gfs2_holder *gh,
fs/gfs2/log.c +1 -1
index a6dd68b458ce..6ee6013fb825 100644
@@ -786,7 +786,7 @@ void gfs2_glock_remove_revoke(struct gfs2_glock *gl)
 {
 	if (atomic_dec_return(&gl->gl_revokes) == 0) {
 		clear_bit(GLF_LFLUSH, &gl->gl_flags);
-		gfs2_glock_queue_put(gl);
+		gfs2_glock_put_async(gl);
 	}
 }
 
fs/gfs2/super.c +2 -2
index 9de789b78bc5..d481db9510ac 100644
@@ -1049,7 +1049,7 @@ static int gfs2_drop_inode(struct inode *inode)
 
 		gfs2_glock_hold(gl);
 		if (!gfs2_queue_try_to_evict(gl))
-			gfs2_glock_queue_put(gl);
+			gfs2_glock_put_async(gl);
 		return 0;
 	}
 
@@ -1255,7 +1255,7 @@ static int gfs2_dinode_dealloc(struct gfs2_inode *ip)
 static void gfs2_glock_put_eventually(struct gfs2_glock *gl)
 {
 	if (current->flags & PF_MEMALLOC)
-		gfs2_glock_queue_put(gl);
+		gfs2_glock_put_async(gl);
 	else
 		gfs2_glock_put(gl);
 }
Signature · 1 line
2.44.0
author maintainer
#9 lore raw

PATCH 9/15 gfs2: Don't set GLF_LOCK in gfs2_dispose_glock_lru

In reply to Andreas Gruenbacher

In gfs2_dispose_glock_lru(), we want to skip glocks which are in the process of transitioning state (as indicated by the set GLF_LOCK flag), but we we don't need to set that flag for requesting a state transition.

---
 fs/gfs2/glock.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
fs/gfs2/glock.c +1 -2
index ed90033b9c72..a700e424f790 100644
@@ -2017,14 +2017,13 @@ __acquires(&lru_lock)
 			atomic_inc(&lru_count);
 			continue;
 		}
-		if (test_and_set_bit(GLF_LOCK, &gl->gl_flags)) {
+		if (test_bit(GLF_LOCK, &gl->gl_flags)) {
 			spin_unlock(&gl->gl_lockref.lock);
 			goto add_back_to_lru;
 		}
 		gl->gl_lockref.count++;
 		if (demote_ok(gl))
 			handle_callback(gl, LM_ST_UNLOCKED, 0, false);
-		WARN_ON(!test_and_clear_bit(GLF_LOCK, &gl->gl_flags));
 		__gfs2_glock_queue_work(gl, 0);
 		spin_unlock(&gl->gl_lockref.lock);
 		cond_resched_lock(&lru_lock);
Signature · 1 line
2.44.0
author maintainer
#10 lore raw

PATCH 10/15 gfs2: Get rid of unnecessary test_and_set_bit

In reply to Andreas Gruenbacher

The GLF_LOCK flag is protected by the gl->gl_lockref.lock spin lock which is held when entering run_queue(), so we can use test_bit() and set_bit() here.

---
 fs/gfs2/glock.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
fs/gfs2/glock.c +2 -1
index a700e424f790..fa047331ea3c 100644
@@ -842,8 +842,9 @@ __acquires(&gl->gl_lockref.lock)
 {
 	struct gfs2_holder *gh = NULL;
 
-	if (test_and_set_bit(GLF_LOCK, &gl->gl_flags))
+	if (test_bit(GLF_LOCK, &gl->gl_flags))
 		return;
+	set_bit(GLF_LOCK, &gl->gl_flags);
 
 	GLOCK_BUG_ON(gl, test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags));
 
Signature · 1 line
2.44.0
author maintainer
#11 lore raw

PATCH 11/15 gfs2: Fix "ignore unlock failures after withdraw"

In reply to Andreas Gruenbacher

Commit 3e11e53041502 tries to suppress dlm_lock() lock conversion errors that occur when the lockspace has already been released.

It does that by setting and checking the SDF_SKIP_DLM_UNLOCK flag. This conflicts with the intended meaning of the SDF_SKIP_DLM_UNLOCK flag, so check whether the lockspace is still allocated instead.

(Given the current DLM API, checking for this kind of error after the fact seems easier that than to make sure that the lockspace is still allocated before calling dlm_lock(). Changing the DLM API so that users maintain the lockspace references themselves would be an option.)

Fixes 3e11e53041502 ("GFS2: ignore unlock failures after withdraw")
---
 fs/gfs2/glock.c | 4 +++-
 fs/gfs2/util.c  | 1 -
 2 files changed, 3 insertions(+), 2 deletions(-)
fs/gfs2/glock.c +3 -1
index fa047331ea3c..ec1f8eb28950 100644
@@ -810,11 +810,13 @@ __acquires(&gl->gl_lockref.lock)
 	}
 
 	if (sdp->sd_lockstruct.ls_ops->lm_lock)	{
+		struct lm_lockstruct *ls = &sdp->sd_lockstruct;
+
 		/* lock_dlm */
 		ret = sdp->sd_lockstruct.ls_ops->lm_lock(gl, target, lck_flags);
 		if (ret == -EINVAL && gl->gl_target == LM_ST_UNLOCKED &&
 		    target == LM_ST_UNLOCKED &&
-		    test_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags)) {
+		    test_bit(DFL_UNMOUNT, &ls->ls_recover_flags)) {
 			finish_xmote(gl, target);
 			gfs2_glock_queue_work(gl, 0);
 		} else if (ret) {
fs/gfs2/util.c +0 -1
index 0535a0206512..09238604d741 100644
@@ -350,7 +350,6 @@ int gfs2_withdraw(struct gfs2_sbd *sdp)
 			fs_err(sdp, "telling LM to unmount\n");
 			lm->lm_unmount(sdp);
 		}
-		set_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags);
 		fs_err(sdp, "File system withdrawn\n");
 		dump_stack();
 		clear_bit(SDF_WITHDRAW_IN_PROG, &sdp->sd_flags);
Signature · 1 line
2.44.0
author maintainer
#12 lore raw

PATCH 12/15 Revert "gfs2: fix glock shrinker ref issues"

In reply to Andreas Gruenbacher

This reverts commit 62862485a4c3a52029fc30f4bdde9af04afdafc9.

Commit 62862485a4c3 tried to fix issues introduced by commit 228804a35caa ("gfs2: Make glock lru list scanning safer"), but like that commit, it failed to account for the bias state_change() adds to the glock reference count for locked glocks. Revert commit 62862485a4c3 so that we can fix commit 228804a35caa properly.

---
 fs/gfs2/glock.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
fs/gfs2/glock.c +1 -3
index ec1f8eb28950..2882a42e88aa 100644
@@ -2056,9 +2056,7 @@ static long gfs2_scan_glock_lru(int nr)
 		if (!test_bit(GLF_LOCK, &gl->gl_flags)) {
 			if (!spin_trylock(&gl->gl_lockref.lock))
 				continue;
-			if (gl->gl_lockref.count <= 1 &&
-			    (gl->gl_state == LM_ST_UNLOCKED ||
-			     demote_ok(gl))) {
+			if (!gl->gl_lockref.count) {
 				list_move(&gl->gl_lru, &dispose);
 				atomic_dec(&lru_count);
 				freed++;
Signature · 1 line
2.44.0
author maintainer
#13 lore raw

PATCH 13/15 gfs2: Fix "Make glock lru list scanning safer"

In reply to Andreas Gruenbacher

Commit 228804a35caa tried to add a refcount check to gfs2_scan_glock_lru() to make sure that glocks that are still referenced cannot be freed. It failed to account for the bias state_change() adds to the refcount for held glocks, so held glocks are no longer removed from the glock cache, which can lead to out-of-memory problems. Fix that. (The inodes those glocks are associated with do get shrunk and do get pushed out of memory.)

In addition, use the same eligibility check in gfs2_scan_glock_lru() and gfs2_dispose_glock_lru().

---
 fs/gfs2/glock.c | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)
fs/gfs2/glock.c +13 -11
index 2882a42e88aa..04e0a8ac61d7 100644
@@ -1987,6 +1987,14 @@ static int glock_cmp(void *priv, const struct list_head *a,
 	return 0;
 }
 
+static bool can_free_glock(struct gfs2_glock *gl)
+{
+	bool held = gl->gl_state != LM_ST_UNLOCKED;
+
+	return !test_bit(GLF_LOCK, &gl->gl_flags) &&
+	       gl->gl_lockref.count == held;
+}
+
 /**
  * gfs2_dispose_glock_lru - Demote a list of glocks
  * @list: The list to dispose of
@@ -2020,7 +2028,7 @@ __acquires(&lru_lock)
 			atomic_inc(&lru_count);
 			continue;
 		}
-		if (test_bit(GLF_LOCK, &gl->gl_flags)) {
+		if (!can_free_glock(gl)) {
 			spin_unlock(&gl->gl_lockref.lock);
 			goto add_back_to_lru;
 		}
@@ -2052,16 +2060,10 @@ static long gfs2_scan_glock_lru(int nr)
 	list_for_each_entry_safe(gl, next, &lru_list, gl_lru) {
 		if (nr-- <= 0)
 			break;
-		/* Test for being demotable */
-		if (!test_bit(GLF_LOCK, &gl->gl_flags)) {
-			if (!spin_trylock(&gl->gl_lockref.lock))
-				continue;
-			if (!gl->gl_lockref.count) {
-				list_move(&gl->gl_lru, &dispose);
-				atomic_dec(&lru_count);
-				freed++;
-			}
-			spin_unlock(&gl->gl_lockref.lock);
+		if (can_free_glock(gl)) {
+			list_move(&gl->gl_lru, &dispose);
+			atomic_dec(&lru_count);
+			freed++;
 		}
 	}
 	if (!list_empty(&dispose))
Signature · 1 line
2.44.0
#14 lore raw

Re: [PATCH 13/15] gfs2: Fix "Make glock lru list scanning safer"

Quoted 21 lines · Andreas Gruenbacher

Commit 228804a35caa tried to add a refcount check to gfs2_scan_glock_lru() to make sure that glocks that are still referenced cannot be freed. It failed to account for the bias state_change() adds to the refcount for held glocks, so held glocks are no longer removed from the glock cache, which can lead to out-of-memory problems. Fix that. (The inodes those glocks are associated with do get shrunk and do get pushed out of memory.)

In addition, use the same eligibility check in gfs2_scan_glock_lru() and gfs2_dispose_glock_lru().

---
  fs/gfs2/glock.c | 24 +++++++++++++-----------
  1 file changed, 13 insertions(+), 11 deletions(-)
fs/gfs2/glock.c +0 -0
index 2882a42e88aa..04e0a8ac61d7 100644

fs/gfs2/glock.c static int glock_cmp(void *priv, const struct list_head *a, · line 1995

Quoted 9 lines
fs/gfs2/glock.c +6 -0
@@ -1987,6 +1987,14 @@ static int glock_cmp(void *priv, const struct list_head *a,
  	return 0;
  }
 
+static bool can_free_glock(struct gfs2_glock *gl)
+{
+	bool held = gl->gl_state != LM_ST_UNLOCKED;
+
+	return !test_bit(GLF_LOCK, &gl->gl_flags) &&
+	       gl->gl_lockref.count == held;

Wouldn't this comparison go wrong when the lockref count is > 1? In any case it's unusual to see a lockref count being compared to a bool.

Andy

Quoted 35 lines
fs/gfs2/glock.c +7 -11
+}
+
  /**
   * gfs2_dispose_glock_lru - Demote a list of glocks
   * @list: The list to dispose of
@@ -2020,7 +2028,7 @@ __acquires(&lru_lock)
  			atomic_inc(&lru_count);
  			continue;
  		}
-		if (test_bit(GLF_LOCK, &gl->gl_flags)) {
+		if (!can_free_glock(gl)) {
  			spin_unlock(&gl->gl_lockref.lock);
  			goto add_back_to_lru;
  		}
@@ -2052,16 +2060,10 @@ static long gfs2_scan_glock_lru(int nr)
  	list_for_each_entry_safe(gl, next, &lru_list, gl_lru) {
  		if (nr-- <= 0)
  			break;
-		/* Test for being demotable */
-		if (!test_bit(GLF_LOCK, &gl->gl_flags)) {
-			if (!spin_trylock(&gl->gl_lockref.lock))
-				continue;
-			if (!gl->gl_lockref.count) {
-				list_move(&gl->gl_lru, &dispose);
-				atomic_dec(&lru_count);
-				freed++;
-			}
-			spin_unlock(&gl->gl_lockref.lock);
+		if (can_free_glock(gl)) {
+			list_move(&gl->gl_lru, &dispose);
+			atomic_dec(&lru_count);
+			freed++;
  		}
  	}
  	if (!list_empty(&dispose))
author maintainer
#15 lore raw

Re: [PATCH 13/15] gfs2: Fix "Make glock lru list scanning safer"

Quoted 25 lines · Andrew Price
Quoted 24 lines · Andreas Gruenbacher

Commit 228804a35caa tried to add a refcount check to gfs2_scan_glock_lru() to make sure that glocks that are still referenced cannot be freed. It failed to account for the bias state_change() adds to the refcount for held glocks, so held glocks are no longer removed from the glock cache, which can lead to out-of-memory problems. Fix that. (The inodes those glocks are associated with do get shrunk and do get pushed out of memory.)

In addition, use the same eligibility check in gfs2_scan_glock_lru() and gfs2_dispose_glock_lru().

---
  fs/gfs2/glock.c | 24 +++++++++++++-----------
  1 file changed, 13 insertions(+), 11 deletions(-)
fs/gfs2/glock.c +0 -0
index 2882a42e88aa..04e0a8ac61d7 100644
@@ -1987,6 +1987,14 @@ static int glock_cmp(void *priv, const struct list_head *a,
      return 0;
  }
 
Quoted 9 lines

fs/gfs2/glock.c static int glock_cmp(void *priv, const struct list_head *a, · line 1995

Quoted 6 lines
fs/gfs2/glock.c +6 -0
+static bool can_free_glock(struct gfs2_glock *gl)
+{
+     bool held = gl->gl_state != LM_ST_UNLOCKED;
+
+     return !test_bit(GLF_LOCK, &gl->gl_flags) &&
+            gl->gl_lockref.count == held;

Wouldn't this comparison go wrong when the lockref count is > 1? In any case it's unusual to see a lockref count being compared to a bool.

We could do something like the below, but the result is the same:

  int count = gl->gl_lockref.count;
  if (gl->gl_state != LM_ST_UNLOCKED)
    count--;
  return !test_bit(GLF_LOCK, &gl->gl_flags) && count == 0;
Quoted 29 lines

Andy

Quoted 27 lines
fs/gfs2/glock.c +3 -10
+}
+
  /**
   * gfs2_dispose_glock_lru - Demote a list of glocks
   * @list: The list to dispose of
@@ -2020,7 +2028,7 @@ __acquires(&lru_lock)
                      atomic_inc(&lru_count);
                      continue;
              }
-             if (test_bit(GLF_LOCK, &gl->gl_flags)) {
+             if (!can_free_glock(gl)) {
                      spin_unlock(&gl->gl_lockref.lock);
                      goto add_back_to_lru;
              }
@@ -2052,16 +2060,10 @@ static long gfs2_scan_glock_lru(int nr)
      list_for_each_entry_safe(gl, next, &lru_list, gl_lru) {
              if (nr-- <= 0)
                      break;
-             /* Test for being demotable */
-             if (!test_bit(GLF_LOCK, &gl->gl_flags)) {
-                     if (!spin_trylock(&gl->gl_lockref.lock))
-                             continue;
-                     if (!gl->gl_lockref.count) {
-                             list_move(&gl->gl_lru, &dispose);
-                             atomic_dec(&lru_count);
-                             freed++;
-                     }
Quoted 9 lines

fs/gfs2/glock.c static long gfs2_scan_glock_lru(int nr) · line 2069

Quoted 8 lines
fs/gfs2/glock.c +4 -1
-                     spin_unlock(&gl->gl_lockref.lock);
+             if (can_free_glock(gl)) {
+                     list_move(&gl->gl_lru, &dispose);
+                     atomic_dec(&lru_count);
+                     freed++;
              }
      }
      if (!list_empty(&dispose))

Thanks,
Andreas

author maintainer
#16 lore raw

PATCH 14/15 gfs2: Get rid of demote_ok checks

In reply to Andreas Gruenbacher

The demote_ok glock operation is only still used to prevent the inode glocks of the "jindex" and "rindex" directories from getting recycled while they are still referenced by sdp->sd_jindex and sdp->sd_rindex. However, the LRU walking code will no longer recycle glocks which are referenced, so the demote_ok glock operation is obsolete and can be removed.

Each of a glock's holders in the gl_holders list is holding a reference on the glock, so when the list of holders isn't empty in demote_ok(), the existing reference count check will already prevent the glock from getting released. This means that demote_ok() is obsolete as well.

---
 Documentation/filesystems/gfs2-glocks.rst |  3 ---
 fs/gfs2/glock.c                           | 25 ++---------------------
 fs/gfs2/glops.c                           | 18 ----------------
 fs/gfs2/incore.h                          |  1 -
 4 files changed, 2 insertions(+), 45 deletions(-)
Documentation/filesystems/gfs2-glocks.rst +0 -3
index 8a5842929b60..fd3bf0f157fb 100644
@@ -61,8 +61,6 @@ Field              Purpose
 go_xmote_th        Called before remote state change (e.g. to sync dirty data)
 go_xmote_bh        Called after remote state change (e.g. to refill cache)
 go_inval           Called if remote state change requires invalidating the cache
-go_demote_ok       Returns boolean value of whether its ok to demote a glock
-                   (e.g. checks timeout, and that there is no cached data)
 go_lock            Called for the first local holder of a lock
 go_unlock          Called on the final local unlock of a lock
 go_dump            Called to print content of object for debugfs file, or on
@@ -96,7 +94,6 @@ Operation        GLF_LOCK bit lock held    gl_lockref.lock spinlock held
 go_xmote_th           Yes                       No
 go_xmote_bh           Yes                       No
 go_inval              Yes                       No
-go_demote_ok          Sometimes                 Yes
 go_lock               Yes                       No
 go_unlock             Yes                       No
 go_dump               Sometimes                 Yes
fs/gfs2/glock.c +2 -23
index 04e0a8ac61d7..5b2fd5994cfc 100644
@@ -192,27 +192,6 @@ struct gfs2_glock *gfs2_glock_hold(struct gfs2_glock *gl)
 	return gl;
 }
 
-/**
- * demote_ok - Check to see if it's ok to unlock a glock
- * @gl: the glock
- *
- * Returns: 1 if it's ok
- */
-
-static int demote_ok(const struct gfs2_glock *gl)
-{
-	const struct gfs2_glock_operations *glops = gl->gl_ops;
-
-	if (gl->gl_state == LM_ST_UNLOCKED)
-		return 0;
-	if (!list_empty(&gl->gl_holders))
-		return 0;
-	if (glops->go_demote_ok)
-		return glops->go_demote_ok(gl);
-	return 1;
-}
-
-
 void gfs2_glock_add_to_lru(struct gfs2_glock *gl)
 {
 	if (!(gl->gl_ops->go_flags & GLOF_LRU))
@@ -1674,7 +1653,7 @@ static void __gfs2_glock_dq(struct gfs2_holder *gh)
 			fast_path = 1;
 	}
 
-	if (!test_bit(GLF_LFLUSH, &gl->gl_flags) && demote_ok(gl))
+	if (!test_bit(GLF_LFLUSH, &gl->gl_flags))
 		gfs2_glock_add_to_lru(gl);
 
 	if (unlikely(!fast_path)) {
@@ -2033,7 +2012,7 @@ __acquires(&lru_lock)
 			goto add_back_to_lru;
 		}
 		gl->gl_lockref.count++;
-		if (demote_ok(gl))
+		if (gl->gl_state != LM_ST_UNLOCKED)
 			handle_callback(gl, LM_ST_UNLOCKED, 0, false);
 		__gfs2_glock_queue_work(gl, 0);
 		spin_unlock(&gl->gl_lockref.lock);
fs/gfs2/glops.c +0 -18
index 68677fb69a73..7013bccb67e8 100644
@@ -385,23 +385,6 @@ static void inode_go_inval(struct gfs2_glock *gl, int flags)
 	gfs2_clear_glop_pending(ip);
 }
 
-/**
- * inode_go_demote_ok - Check to see if it's ok to unlock an inode glock
- * @gl: the glock
- *
- * Returns: 1 if it's ok
- */
-
-static int inode_go_demote_ok(const struct gfs2_glock *gl)
-{
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
-
-	if (sdp->sd_jindex == gl->gl_object || sdp->sd_rindex == gl->gl_object)
-		return 0;
-
-	return 1;
-}
-
 static int gfs2_dinode_in(struct gfs2_inode *ip, const void *buf)
 {
 	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
@@ -722,7 +705,6 @@ const struct gfs2_glock_operations gfs2_meta_glops = {
 const struct gfs2_glock_operations gfs2_inode_glops = {
 	.go_sync = inode_go_sync,
 	.go_inval = inode_go_inval,
-	.go_demote_ok = inode_go_demote_ok,
 	.go_instantiate = inode_go_instantiate,
 	.go_held = inode_go_held,
 	.go_dump = inode_go_dump,
fs/gfs2/incore.h +0 -1
index 95a334d64da2..2c4c0ee2c014 100644
@@ -218,7 +218,6 @@ struct gfs2_glock_operations {
 	int (*go_sync) (struct gfs2_glock *gl);
 	int (*go_xmote_bh)(struct gfs2_glock *gl);
 	void (*go_inval) (struct gfs2_glock *gl, int flags);
-	int (*go_demote_ok) (const struct gfs2_glock *gl);
 	int (*go_instantiate) (struct gfs2_glock *gl);
 	int (*go_held)(struct gfs2_holder *gh);
 	void (*go_dump)(struct seq_file *seq, const struct gfs2_glock *gl,
Signature · 1 line
2.44.0
author maintainer
#17 lore raw

Re: [PATCH 14/15] gfs2: Get rid of demote_ok checks

Quoted 11 lines · Andreas Gruenbacher

The demote_ok glock operation is only still used to prevent the inode glocks of the "jindex" and "rindex" directories from getting recycled while they are still referenced by sdp->sd_jindex and sdp->sd_rindex. However, the LRU walking code will no longer recycle glocks which are referenced, so the demote_ok glock operation is obsolete and can be removed.

Each of a glock's holders in the gl_holders list is holding a reference on the glock, so when the list of holders isn't empty in demote_ok(), the existing reference count check will already prevent the glock from getting released. This means that demote_ok() is obsolete as well.

This patch only works after a few additional patches I haven't posted yet. I'm removing it from for-next for now and I'll repost later.

Thanks,
Andreas

author maintainer
#18 lore raw

PATCH 15/15 gfs2: Fix lru_count accounting

In reply to Andreas Gruenbacher

Currently, gfs2_scan_glock_lru() decrements lru_count when a glock is moved onto the dispose list. When such a glock is then stolen from the dispose list while gfs2_dispose_glock_lru() doesn't hold the lru_lock, lru_count will be decremented again, so the counter will eventually go negative.

This bug has existed in one form or another since at least commit 97cc1025b1a91 ("GFS2: Kill two daemons with one patch").

Fix this by only decrementing lru_count when we actually remove a glock and schedule for it to be unlocked and dropped. We also don't need to remove and then re-add glocks when we can just as well move them back onto the lru_list when necessary.

In addition, return the number of glocks freed as we should, not the number of glocks moved onto the dispose list.

---
 fs/gfs2/glock.c | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)
fs/gfs2/glock.c +10 -11
index 5b2fd5994cfc..4be140ab53e4 100644
@@ -1988,29 +1988,30 @@ static bool can_free_glock(struct gfs2_glock *gl)
  * private)
  */
 
-static void gfs2_dispose_glock_lru(struct list_head *list)
+static long gfs2_dispose_glock_lru(struct list_head *list)
 __releases(&lru_lock)
 __acquires(&lru_lock)
 {
 	struct gfs2_glock *gl;
+	long freed = 0;
 
 	list_sort(NULL, list, glock_cmp);
 
 	while(!list_empty(list)) {
 		gl = list_first_entry(list, struct gfs2_glock, gl_lru);
-		list_del_init(&gl->gl_lru);
-		clear_bit(GLF_LRU, &gl->gl_flags);
 		if (!spin_trylock(&gl->gl_lockref.lock)) {
 add_back_to_lru:
-			list_add(&gl->gl_lru, &lru_list);
-			set_bit(GLF_LRU, &gl->gl_flags);
-			atomic_inc(&lru_count);
+			list_move(&gl->gl_lru, &lru_list);
 			continue;
 		}
 		if (!can_free_glock(gl)) {
 			spin_unlock(&gl->gl_lockref.lock);
 			goto add_back_to_lru;
 		}
+		list_del_init(&gl->gl_lru);
+		atomic_dec(&lru_count);
+		clear_bit(GLF_LRU, &gl->gl_flags);
+		freed++;
 		gl->gl_lockref.count++;
 		if (gl->gl_state != LM_ST_UNLOCKED)
 			handle_callback(gl, LM_ST_UNLOCKED, 0, false);
@@ -2018,6 +2019,7 @@ __acquires(&lru_lock)
 		spin_unlock(&gl->gl_lockref.lock);
 		cond_resched_lock(&lru_lock);
 	}
+	return freed;
 }
 
 /**
@@ -2039,14 +2041,11 @@ static long gfs2_scan_glock_lru(int nr)
 	list_for_each_entry_safe(gl, next, &lru_list, gl_lru) {
 		if (nr-- <= 0)
 			break;
-		if (can_free_glock(gl)) {
+		if (can_free_glock(gl))
 			list_move(&gl->gl_lru, &dispose);
-			atomic_dec(&lru_count);
-			freed++;
-		}
 	}
 	if (!list_empty(&dispose))
-		gfs2_dispose_glock_lru(&dispose);
+		freed = gfs2_dispose_glock_lru(&dispose);
 	spin_unlock(&lru_lock);
 
 	return freed;
Signature · 1 line
2.44.0