Skip to contentSkip to conversation

ceph: check session state after bumping session->s_seq

PATCH Superseded Jeff Layton 2 participants 2 messages Oct 12, 2020 Ceph client

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 ceph: check session state after bumping session->s_seq

Some messages sent by the MDS entail a session sequence number increment, and the MDS will drop certain types of requests on the floor when the sequence numbers don't match.

In particular, a REQUEST_CLOSE message can cross with one of the sequence morphing messages from the MDS which can cause the client to stall, waiting for a response that will never come.

Originally, this meant an up to 5s delay before the recurring workqueue job kicked in and resent the request, but a recent change made it so that the client would never resend, causing a 60s stall unmounting and sometimes a blockisting event.

Add a new helper for incrementing the session sequence and then testing to see whether a REQUEST_CLOSE needs to be resent. Change all of the bare sequence counter increments to use the new helper.

URL: https://tracker.ceph.com/issues/47563

Fixes fa9967734227 ("ceph: fix potential mdsc use-after-free crash")
---
 fs/ceph/caps.c       |  2 +-
 fs/ceph/mds_client.c | 35 +++++++++++++++++++++++++++++------
 fs/ceph/mds_client.h |  1 +
 fs/ceph/quota.c      |  2 +-
 fs/ceph/snap.c       |  2 +-
 5 files changed, 33 insertions(+), 9 deletions(-)

v2: move seq increment and check for closed session into new helper

fs/ceph/caps.c +1 -1
index c00abd7eefc1..ba0e4f44862c 100644
@@ -4071,7 +4071,7 @@ void ceph_handle_caps(struct ceph_mds_session *session,
 	     vino.snap, inode);
 
 	mutex_lock(&session->s_mutex);
-	session->s_seq++;
+	inc_session_sequence(session);
 	dout(" mds%d seq %lld cap seq %u\n", session->s_mds, session->s_seq,
 	     (unsigned)seq);
 
fs/ceph/mds_client.c +29 -6
index 0190555b1f9e..17b94f06826a 100644
@@ -4237,7 +4237,7 @@ static void handle_lease(struct ceph_mds_client *mdsc,
 	     dname.len, dname.name);
 
 	mutex_lock(&session->s_mutex);
-	session->s_seq++;
+	inc_session_sequence(session);
 
 	if (!inode) {
 		dout("handle_lease no inode %llx\n", vino.ino);
@@ -4384,14 +4384,25 @@ static void maybe_recover_session(struct ceph_mds_client *mdsc)
 	ceph_force_reconnect(fsc->sb);
 }
 
+static bool check_session_closing(struct ceph_mds_session *s)
+{
+	int ret;
+
+	if (s->s_state != CEPH_MDS_SESSION_CLOSING)
+		return true;
+
+	dout("resending session close request for mds%d\n", s->s_mds);
+	ret = request_close_session(s);
+	if (ret < 0)
+		pr_err("ceph: Unable to close session to mds %d: %d\n", s->s_mds, ret);
+	return false;
+}
+
 bool check_session_state(struct ceph_mds_session *s)
 {
-	if (s->s_state == CEPH_MDS_SESSION_CLOSING) {
-		dout("resending session close request for mds%d\n",
-				s->s_mds);
-		request_close_session(s);
+	if (!check_session_closing(s))
 		return false;
-	}
+
 	if (s->s_ttl && time_after(jiffies, s->s_ttl)) {
 		if (s->s_state == CEPH_MDS_SESSION_OPEN) {
 			s->s_state = CEPH_MDS_SESSION_HUNG;
@@ -4408,6 +4419,18 @@ bool check_session_state(struct ceph_mds_session *s)
 	return true;
 }
 
+/*
+ * If the sequence is incremented while we're waiting on a REQUEST_CLOSE reply,
+ * then we need to retransmit that request.
+ */
+void inc_session_sequence(struct ceph_mds_session *s)
+{
+	lockdep_assert_held(&s->s_mutex);
+
+	s->s_seq++;
+	check_session_closing(s);
+}
+
 /*
  * delayed work -- periodically trim expired leases, renew caps with mds
  */
fs/ceph/mds_client.h +1 -0
index cbf8af437140..f5adbebcb38e 100644
@@ -480,6 +480,7 @@ struct ceph_mds_client {
 extern const char *ceph_mds_op_name(int op);
 
 extern bool check_session_state(struct ceph_mds_session *s);
+void inc_session_sequence(struct ceph_mds_session *s);
 
 extern struct ceph_mds_session *
 __ceph_lookup_mds_session(struct ceph_mds_client *, int mds);
fs/ceph/quota.c +1 -1
index 83cb4f26b689..9b785f11e95a 100644
@@ -53,7 +53,7 @@ void ceph_handle_quota(struct ceph_mds_client *mdsc,
 
 	/* increment msg sequence number */
 	mutex_lock(&session->s_mutex);
-	session->s_seq++;
+	inc_session_sequence(session);
 	mutex_unlock(&session->s_mutex);
 
 	/* lookup inode */
fs/ceph/snap.c +1 -1
index 0da39c16dab4..b611f829cb61 100644
@@ -873,7 +873,7 @@ void ceph_handle_snap(struct ceph_mds_client *mdsc,
 	     ceph_snap_op_name(op), split, trace_len);
 
 	mutex_lock(&session->s_mutex);
-	session->s_seq++;
+	inc_session_sequence(session);
 	mutex_unlock(&session->s_mutex);
 
 	down_write(&mdsc->snap_rwsem);
Signature · 1 line
2.26.2
maintainer
#1 lore raw
Quoted 73 lines · Jeff Layton

Some messages sent by the MDS entail a session sequence number increment, and the MDS will drop certain types of requests on the floor when the sequence numbers don't match.

In particular, a REQUEST_CLOSE message can cross with one of the sequence morphing messages from the MDS which can cause the client to stall, waiting for a response that will never come.

Originally, this meant an up to 5s delay before the recurring workqueue job kicked in and resent the request, but a recent change made it so that the client would never resend, causing a 60s stall unmounting and sometimes a blockisting event.

Add a new helper for incrementing the session sequence and then testing to see whether a REQUEST_CLOSE needs to be resent. Change all of the bare sequence counter increments to use the new helper.

URL: https://tracker.ceph.com/issues/47563

Fixes fa9967734227 ("ceph: fix potential mdsc use-after-free crash")
---
 fs/ceph/caps.c       |  2 +-
 fs/ceph/mds_client.c | 35 +++++++++++++++++++++++++++++------
 fs/ceph/mds_client.h |  1 +
 fs/ceph/quota.c      |  2 +-
 fs/ceph/snap.c       |  2 +-
 5 files changed, 33 insertions(+), 9 deletions(-)

v2: move seq increment and check for closed session into new helper

fs/ceph/caps.c +1 -1
index c00abd7eefc1..ba0e4f44862c 100644
@@ -4071,7 +4071,7 @@ void ceph_handle_caps(struct ceph_mds_session *session,
             vino.snap, inode);
 
        mutex_lock(&session->s_mutex);
-       session->s_seq++;
+       inc_session_sequence(session);
        dout(" mds%d seq %lld cap seq %u\n", session->s_mds, session->s_seq,
             (unsigned)seq);
 
fs/ceph/mds_client.c +12 -1
index 0190555b1f9e..17b94f06826a 100644
@@ -4237,7 +4237,7 @@ static void handle_lease(struct ceph_mds_client *mdsc,
             dname.len, dname.name);
 
        mutex_lock(&session->s_mutex);
-       session->s_seq++;
+       inc_session_sequence(session);
 
        if (!inode) {
                dout("handle_lease no inode %llx\n", vino.ino);
@@ -4384,14 +4384,25 @@ static void maybe_recover_session(struct ceph_mds_client *mdsc)
        ceph_force_reconnect(fsc->sb);
 }
 
+static bool check_session_closing(struct ceph_mds_session *s)
+{
+       int ret;
+
+       if (s->s_state != CEPH_MDS_SESSION_CLOSING)
+               return true;
+
+       dout("resending session close request for mds%d\n", s->s_mds);
+       ret = request_close_session(s);
+       if (ret < 0)
+               pr_err("ceph: Unable to close session to mds %d: %d\n", s->s_mds, ret);

fs/ceph/mds_client.c static void maybe_recover_session(struct ceph_mds_client *mdsc) · line 4403

Quoted 10 lines
fs/ceph/mds_client.c +4 -4
+       return false;
+}
+
 bool check_session_state(struct ceph_mds_session *s)
 {
-       if (s->s_state == CEPH_MDS_SESSION_CLOSING) {
-               dout("resending session close request for mds%d\n",
-                               s->s_mds);
-               request_close_session(s);
+       if (!check_session_closing(s))

Do we actually need it in check_session_state() now? Given that it is conditioned on CEPH_MDS_SESSION_CLOSING and we know check_session_state() doesn't get called in that case, it seems like dead code to me.

Thanks,

Ilya