Skip to contentSkip to conversation

ceph: fix NULL pointer dereference for req->r_session

PATCH Superseded [email protected] 2 participants 5 messages Nov 8, 2022 – Nov 10, 2022

Subsystems (2)

v3 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

PATCH v3 1/1 ceph: fix NULL pointer dereference for req->r_session

From: Xiubo Li <[email protected]>

The request's r_session maybe changed when it was forwarded or resent.

Cc: [email protected]
URL: https://bugzilla.redhat.com/show_bug.cgi?id=2137955

---
 fs/ceph/caps.c | 60 ++++++++++++++++----------------------------------
 1 file changed, 19 insertions(+), 41 deletions(-)
fs/ceph/caps.c +19 -41
index 894adfb4a092..83f9e18e3169 100644
@@ -2297,8 +2297,10 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
 	struct ceph_mds_client *mdsc = ceph_sb_to_client(inode->i_sb)->mdsc;
 	struct ceph_inode_info *ci = ceph_inode(inode);
 	struct ceph_mds_request *req1 = NULL, *req2 = NULL;
+	struct ceph_mds_session **sessions = NULL;
+	struct ceph_mds_session *s;
 	unsigned int max_sessions;
-	int ret, err = 0;
+	int i, ret, err = 0;
 
 	spin_lock(&ci->i_unsafe_lock);
 	if (S_ISDIR(inode->i_mode) && !list_empty(&ci->i_unsafe_dirops)) {
@@ -2315,28 +2317,19 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
 	}
 	spin_unlock(&ci->i_unsafe_lock);
 
-	/*
-	 * The mdsc->max_sessions is unlikely to be changed
-	 * mostly, here we will retry it by reallocating the
-	 * sessions array memory to get rid of the mdsc->mutex
-	 * lock.
-	 */
-retry:
-	max_sessions = mdsc->max_sessions;
-
 	/*
 	 * Trigger to flush the journal logs in all the relevant MDSes
 	 * manually, or in the worst case we must wait at most 5 seconds
 	 * to wait the journal logs to be flushed by the MDSes periodically.
 	 */
-	if ((req1 || req2) && likely(max_sessions)) {
-		struct ceph_mds_session **sessions = NULL;
-		struct ceph_mds_session *s;
+	mutex_lock(&mdsc->mutex);
+	max_sessions = mdsc->max_sessions;
+	if (req1 || req2) {
 		struct ceph_mds_request *req;
-		int i;
 
 		sessions = kcalloc(max_sessions, sizeof(s), GFP_KERNEL);
 		if (!sessions) {
+			mutex_unlock(&mdsc->mutex);
 			err = -ENOMEM;
 			goto out;
 		}
@@ -2346,18 +2339,8 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
 			list_for_each_entry(req, &ci->i_unsafe_dirops,
 					    r_unsafe_dir_item) {
 				s = req->r_session;
-				if (!s)
+				if (!s || unlikely(s->s_mds >= max_sessions))
 					continue;
-				if (unlikely(s->s_mds >= max_sessions)) {
-					spin_unlock(&ci->i_unsafe_lock);
-					for (i = 0; i < max_sessions; i++) {
-						s = sessions[i];
-						if (s)
-							ceph_put_mds_session(s);
-					}
-					kfree(sessions);
-					goto retry;
-				}
 				if (!sessions[s->s_mds]) {
 					s = ceph_get_mds_session(s);
 					sessions[s->s_mds] = s;
@@ -2368,18 +2351,8 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
 			list_for_each_entry(req, &ci->i_unsafe_iops,
 					    r_unsafe_target_item) {
 				s = req->r_session;
-				if (!s)
+				if (!s || unlikely(s->s_mds >= max_sessions))
 					continue;
-				if (unlikely(s->s_mds >= max_sessions)) {
-					spin_unlock(&ci->i_unsafe_lock);
-					for (i = 0; i < max_sessions; i++) {
-						s = sessions[i];
-						if (s)
-							ceph_put_mds_session(s);
-					}
-					kfree(sessions);
-					goto retry;
-				}
 				if (!sessions[s->s_mds]) {
 					s = ceph_get_mds_session(s);
 					sessions[s->s_mds] = s;
@@ -2391,13 +2364,18 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
 		/* the auth MDS */
 		spin_lock(&ci->i_ceph_lock);
 		if (ci->i_auth_cap) {
-		      s = ci->i_auth_cap->session;
-		      if (!sessions[s->s_mds])
-			      sessions[s->s_mds] = ceph_get_mds_session(s);
+			s = ci->i_auth_cap->session;
+			if (likely(s->s_mds < max_sessions)
+			    && !sessions[s->s_mds]) {
+				sessions[s->s_mds] = ceph_get_mds_session(s);
+			}
 		}
 		spin_unlock(&ci->i_ceph_lock);
+	}
+	mutex_unlock(&mdsc->mutex);
 
-		/* send flush mdlog request to MDSes */
+	/* send flush mdlog request to MDSes */
+	if (sessions) {
 		for (i = 0; i < max_sessions; i++) {
 			s = sessions[i];
 			if (s) {
@@ -2405,7 +2383,6 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
 				ceph_put_mds_session(s);
 			}
 		}
-		kfree(sessions);
 	}
 
 	dout("%s %p wait on tid %llu %llu\n", __func__,
@@ -2424,6 +2401,7 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
 	}
 
 out:
+	kfree(sessions);
 	if (req1)
 		ceph_mdsc_put_request(req1);
 	if (req2)
Signature · 1 line
2.31.1
maintainer
#1 lore raw
Quoted 103 lines · On Tue, Nov 8, 2022 at 2:56 PM <[email protected]> wrote:

From: Xiubo Li <[email protected]>

The request's r_session maybe changed when it was forwarded or resent.

Cc: [email protected]
URL: https://bugzilla.redhat.com/show_bug.cgi?id=2137955

---
 fs/ceph/caps.c | 60 ++++++++++++++++----------------------------------
 1 file changed, 19 insertions(+), 41 deletions(-)
fs/ceph/caps.c +9 -36
index 894adfb4a092..83f9e18e3169 100644
@@ -2297,8 +2297,10 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
        struct ceph_mds_client *mdsc = ceph_sb_to_client(inode->i_sb)->mdsc;
        struct ceph_inode_info *ci = ceph_inode(inode);
        struct ceph_mds_request *req1 = NULL, *req2 = NULL;
+       struct ceph_mds_session **sessions = NULL;
+       struct ceph_mds_session *s;
        unsigned int max_sessions;
-       int ret, err = 0;
+       int i, ret, err = 0;
 
        spin_lock(&ci->i_unsafe_lock);
        if (S_ISDIR(inode->i_mode) && !list_empty(&ci->i_unsafe_dirops)) {
@@ -2315,28 +2317,19 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
        }
        spin_unlock(&ci->i_unsafe_lock);
 
-       /*
-        * The mdsc->max_sessions is unlikely to be changed
-        * mostly, here we will retry it by reallocating the
-        * sessions array memory to get rid of the mdsc->mutex
-        * lock.
-        */
-retry:
-       max_sessions = mdsc->max_sessions;
-
        /*
         * Trigger to flush the journal logs in all the relevant MDSes
         * manually, or in the worst case we must wait at most 5 seconds
         * to wait the journal logs to be flushed by the MDSes periodically.
         */
-       if ((req1 || req2) && likely(max_sessions)) {
-               struct ceph_mds_session **sessions = NULL;
-               struct ceph_mds_session *s;
+       mutex_lock(&mdsc->mutex);
+       max_sessions = mdsc->max_sessions;
+       if (req1 || req2) {
                struct ceph_mds_request *req;
-               int i;
 
                sessions = kcalloc(max_sessions, sizeof(s), GFP_KERNEL);
                if (!sessions) {
+                       mutex_unlock(&mdsc->mutex);
                        err = -ENOMEM;
                        goto out;
                }
@@ -2346,18 +2339,8 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
                        list_for_each_entry(req, &ci->i_unsafe_dirops,
                                            r_unsafe_dir_item) {
                                s = req->r_session;
-                               if (!s)
+                               if (!s || unlikely(s->s_mds >= max_sessions))
                                        continue;
-                               if (unlikely(s->s_mds >= max_sessions)) {
-                                       spin_unlock(&ci->i_unsafe_lock);
-                                       for (i = 0; i < max_sessions; i++) {
-                                               s = sessions[i];
-                                               if (s)
-                                                       ceph_put_mds_session(s);
-                                       }
-                                       kfree(sessions);
-                                       goto retry;
-                               }
                                if (!sessions[s->s_mds]) {
                                        s = ceph_get_mds_session(s);
                                        sessions[s->s_mds] = s;
@@ -2368,18 +2351,8 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
                        list_for_each_entry(req, &ci->i_unsafe_iops,
                                            r_unsafe_target_item) {
                                s = req->r_session;
-                               if (!s)
+                               if (!s || unlikely(s->s_mds >= max_sessions))
                                        continue;
-                               if (unlikely(s->s_mds >= max_sessions)) {
-                                       spin_unlock(&ci->i_unsafe_lock);
-                                       for (i = 0; i < max_sessions; i++) {
-                                               s = sessions[i];
-                                               if (s)
-                                                       ceph_put_mds_session(s);
-                                       }
-                                       kfree(sessions);
-                                       goto retry;
-                               }
                                if (!sessions[s->s_mds]) {
                                        s = ceph_get_mds_session(s);
                                        sessions[s->s_mds] = s;

fs/ceph/caps.c static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode) · line 2369

Quoted 9 lines
fs/ceph/caps.c +3 -3
@@ -2391,13 +2364,18 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
                /* the auth MDS */
                spin_lock(&ci->i_ceph_lock);
                if (ci->i_auth_cap) {
-                     s = ci->i_auth_cap->session;
-                     if (!sessions[s->s_mds])
-                             sessions[s->s_mds] = ceph_get_mds_session(s);
+                       s = ci->i_auth_cap->session;
+                       if (likely(s->s_mds < max_sessions)
+                           && !sessions[s->s_mds]) {

Hi Xiubo,

Nit: keep && on the previous line for style consistency.

fs/ceph/caps.c static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode) · line 2378

Quoted 10 lines
fs/ceph/caps.c +6 -1
+                               sessions[s->s_mds] = ceph_get_mds_session(s);
+                       }
                }
                spin_unlock(&ci->i_ceph_lock);
+       }
+       mutex_unlock(&mdsc->mutex);
 
-               /* send flush mdlog request to MDSes */
+       /* send flush mdlog request to MDSes */
+       if (sessions) {

Since mdlog is flushed only in "if (req1 || req2)" case, why not keep max_sessions loop there and avoid sessions != NULL check?

Then, you could also move mdsc->mutex acquisition and max_sessions assignment into "if (req1 || req2)" branch and keep sessions, s and i declarations where there are today.

Thanks,

Ilya

author
#2 lore raw
Quoted 114 lines · Ilya Dryomov
Quoted 102 lines · On Tue, Nov 8, 2022 at 2:56 PM <[email protected]> wrote:

From: Xiubo Li <[email protected]>

The request's r_session maybe changed when it was forwarded or resent.

Cc: [email protected]
URL: https://bugzilla.redhat.com/show_bug.cgi?id=2137955

---
  fs/ceph/caps.c | 60 ++++++++++++++++----------------------------------
  1 file changed, 19 insertions(+), 41 deletions(-)
fs/ceph/caps.c +9 -36
index 894adfb4a092..83f9e18e3169 100644
@@ -2297,8 +2297,10 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
         struct ceph_mds_client *mdsc = ceph_sb_to_client(inode->i_sb)->mdsc;
         struct ceph_inode_info *ci = ceph_inode(inode);
         struct ceph_mds_request *req1 = NULL, *req2 = NULL;
+       struct ceph_mds_session **sessions = NULL;
+       struct ceph_mds_session *s;
         unsigned int max_sessions;
-       int ret, err = 0;
+       int i, ret, err = 0;
 
         spin_lock(&ci->i_unsafe_lock);
         if (S_ISDIR(inode->i_mode) && !list_empty(&ci->i_unsafe_dirops)) {
@@ -2315,28 +2317,19 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
         }
         spin_unlock(&ci->i_unsafe_lock);
 
-       /*
-        * The mdsc->max_sessions is unlikely to be changed
-        * mostly, here we will retry it by reallocating the
-        * sessions array memory to get rid of the mdsc->mutex
-        * lock.
-        */
-retry:
-       max_sessions = mdsc->max_sessions;
-
         /*
          * Trigger to flush the journal logs in all the relevant MDSes
          * manually, or in the worst case we must wait at most 5 seconds
          * to wait the journal logs to be flushed by the MDSes periodically.
          */
-       if ((req1 || req2) && likely(max_sessions)) {
-               struct ceph_mds_session **sessions = NULL;
-               struct ceph_mds_session *s;
+       mutex_lock(&mdsc->mutex);
+       max_sessions = mdsc->max_sessions;
+       if (req1 || req2) {
                 struct ceph_mds_request *req;
-               int i;
 
                 sessions = kcalloc(max_sessions, sizeof(s), GFP_KERNEL);
                 if (!sessions) {
+                       mutex_unlock(&mdsc->mutex);
                         err = -ENOMEM;
                         goto out;
                 }
@@ -2346,18 +2339,8 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
                         list_for_each_entry(req, &ci->i_unsafe_dirops,
                                             r_unsafe_dir_item) {
                                 s = req->r_session;
-                               if (!s)
+                               if (!s || unlikely(s->s_mds >= max_sessions))
                                         continue;
-                               if (unlikely(s->s_mds >= max_sessions)) {
-                                       spin_unlock(&ci->i_unsafe_lock);
-                                       for (i = 0; i < max_sessions; i++) {
-                                               s = sessions[i];
-                                               if (s)
-                                                       ceph_put_mds_session(s);
-                                       }
-                                       kfree(sessions);
-                                       goto retry;
-                               }
                                 if (!sessions[s->s_mds]) {
                                         s = ceph_get_mds_session(s);
                                         sessions[s->s_mds] = s;
@@ -2368,18 +2351,8 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
                         list_for_each_entry(req, &ci->i_unsafe_iops,
                                             r_unsafe_target_item) {
                                 s = req->r_session;
-                               if (!s)
+                               if (!s || unlikely(s->s_mds >= max_sessions))
                                         continue;
-                               if (unlikely(s->s_mds >= max_sessions)) {
-                                       spin_unlock(&ci->i_unsafe_lock);
-                                       for (i = 0; i < max_sessions; i++) {
-                                               s = sessions[i];
-                                               if (s)
-                                                       ceph_put_mds_session(s);
-                                       }
-                                       kfree(sessions);
-                                       goto retry;
-                               }
                                 if (!sessions[s->s_mds]) {
                                         s = ceph_get_mds_session(s);
                                         sessions[s->s_mds] = s;

fs/ceph/caps.c static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode) · line 2369

Quoted 9 lines
fs/ceph/caps.c +3 -3
@@ -2391,13 +2364,18 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
                 /* the auth MDS */
                 spin_lock(&ci->i_ceph_lock);
                 if (ci->i_auth_cap) {
-                     s = ci->i_auth_cap->session;
-                     if (!sessions[s->s_mds])
-                             sessions[s->s_mds] = ceph_get_mds_session(s);
+                       s = ci->i_auth_cap->session;
+                       if (likely(s->s_mds < max_sessions)
+                           && !sessions[s->s_mds]) {

Hi Xiubo,

Quoted 1 line

Nit: keep && on the previous line for style consistency.

Sure. Will fix it.

Quoted 8 lines
Quoted 8 lines
fs/ceph/caps.c +4 -0
+                               sessions[s->s_mds] = ceph_get_mds_session(s);
+                       }
                 }
                 spin_unlock(&ci->i_ceph_lock);
+       }
+       mutex_unlock(&mdsc->mutex);
 
Quoted 5 lines

fs/ceph/caps.c static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode) · line 2378

Quoted 3 lines
fs/ceph/caps.c +2 -1
-               /* send flush mdlog request to MDSes */
+       /* send flush mdlog request to MDSes */
+       if (sessions) {

Since mdlog is flushed only in "if (req1 || req2)" case, why not keep max_sessions loop there and avoid sessions != NULL check?

This is because I must drop the mdsc->mutex before calling "send_flush_mdlog()" in the max_sessions loop.

Thanks!

- Xiubo

Quoted 9 lines

Then, you could also move mdsc->mutex acquisition and max_sessions assignment into "if (req1 || req2)" branch and keep sessions, s and i declarations where there are today.

Thanks,

Ilya

maintainer
#3 lore raw
Quoted 130 lines · Xiubo Li
Quoted 114 lines · Ilya Dryomov
Quoted 102 lines · On Tue, Nov 8, 2022 at 2:56 PM <[email protected]> wrote:

From: Xiubo Li <[email protected]>

The request's r_session maybe changed when it was forwarded or resent.

Cc: [email protected]
URL: https://bugzilla.redhat.com/show_bug.cgi?id=2137955

---
  fs/ceph/caps.c | 60 ++++++++++++++++----------------------------------
  1 file changed, 19 insertions(+), 41 deletions(-)
fs/ceph/caps.c +9 -36
index 894adfb4a092..83f9e18e3169 100644
@@ -2297,8 +2297,10 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
         struct ceph_mds_client *mdsc = ceph_sb_to_client(inode->i_sb)->mdsc;
         struct ceph_inode_info *ci = ceph_inode(inode);
         struct ceph_mds_request *req1 = NULL, *req2 = NULL;
+       struct ceph_mds_session **sessions = NULL;
+       struct ceph_mds_session *s;
         unsigned int max_sessions;
-       int ret, err = 0;
+       int i, ret, err = 0;
 
         spin_lock(&ci->i_unsafe_lock);
         if (S_ISDIR(inode->i_mode) && !list_empty(&ci->i_unsafe_dirops)) {
@@ -2315,28 +2317,19 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
         }
         spin_unlock(&ci->i_unsafe_lock);
 
-       /*
-        * The mdsc->max_sessions is unlikely to be changed
-        * mostly, here we will retry it by reallocating the
-        * sessions array memory to get rid of the mdsc->mutex
-        * lock.
-        */
-retry:
-       max_sessions = mdsc->max_sessions;
-
         /*
          * Trigger to flush the journal logs in all the relevant MDSes
          * manually, or in the worst case we must wait at most 5 seconds
          * to wait the journal logs to be flushed by the MDSes periodically.
          */
-       if ((req1 || req2) && likely(max_sessions)) {
-               struct ceph_mds_session **sessions = NULL;
-               struct ceph_mds_session *s;
+       mutex_lock(&mdsc->mutex);
+       max_sessions = mdsc->max_sessions;
+       if (req1 || req2) {
                 struct ceph_mds_request *req;
-               int i;
 
                 sessions = kcalloc(max_sessions, sizeof(s), GFP_KERNEL);
                 if (!sessions) {
+                       mutex_unlock(&mdsc->mutex);
                         err = -ENOMEM;
                         goto out;
                 }
@@ -2346,18 +2339,8 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
                         list_for_each_entry(req, &ci->i_unsafe_dirops,
                                             r_unsafe_dir_item) {
                                 s = req->r_session;
-                               if (!s)
+                               if (!s || unlikely(s->s_mds >= max_sessions))
                                         continue;
-                               if (unlikely(s->s_mds >= max_sessions)) {
-                                       spin_unlock(&ci->i_unsafe_lock);
-                                       for (i = 0; i < max_sessions; i++) {
-                                               s = sessions[i];
-                                               if (s)
-                                                       ceph_put_mds_session(s);
-                                       }
-                                       kfree(sessions);
-                                       goto retry;
-                               }
                                 if (!sessions[s->s_mds]) {
                                         s = ceph_get_mds_session(s);
                                         sessions[s->s_mds] = s;
@@ -2368,18 +2351,8 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
                         list_for_each_entry(req, &ci->i_unsafe_iops,
                                             r_unsafe_target_item) {
                                 s = req->r_session;
-                               if (!s)
+                               if (!s || unlikely(s->s_mds >= max_sessions))
                                         continue;
-                               if (unlikely(s->s_mds >= max_sessions)) {
-                                       spin_unlock(&ci->i_unsafe_lock);
-                                       for (i = 0; i < max_sessions; i++) {
-                                               s = sessions[i];
-                                               if (s)
-                                                       ceph_put_mds_session(s);
-                                       }
-                                       kfree(sessions);
-                                       goto retry;
-                               }
                                 if (!sessions[s->s_mds]) {
                                         s = ceph_get_mds_session(s);
                                         sessions[s->s_mds] = s;

fs/ceph/caps.c static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode) · line 2369

Quoted 9 lines
fs/ceph/caps.c +3 -3
@@ -2391,13 +2364,18 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
                 /* the auth MDS */
                 spin_lock(&ci->i_ceph_lock);
                 if (ci->i_auth_cap) {
-                     s = ci->i_auth_cap->session;
-                     if (!sessions[s->s_mds])
-                             sessions[s->s_mds] = ceph_get_mds_session(s);
+                       s = ci->i_auth_cap->session;
+                       if (likely(s->s_mds < max_sessions)
+                           && !sessions[s->s_mds]) {

Hi Xiubo,

Quoted 1 line

Nit: keep && on the previous line for style consistency.

Sure. Will fix it.

Quoted 8 lines
Quoted 8 lines
fs/ceph/caps.c +4 -0
+                               sessions[s->s_mds] = ceph_get_mds_session(s);
+                       }
                 }
                 spin_unlock(&ci->i_ceph_lock);
+       }
+       mutex_unlock(&mdsc->mutex);
 
Quoted 8 lines
Quoted 5 lines

fs/ceph/caps.c static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode) · line 2378

Quoted 3 lines
fs/ceph/caps.c +2 -1
-               /* send flush mdlog request to MDSes */
+       /* send flush mdlog request to MDSes */
+       if (sessions) {

Since mdlog is flushed only in "if (req1 || req2)" case, why not keep max_sessions loop there and avoid sessions != NULL check?

This is because I must drop the mdsc->mutex before calling "send_flush_mdlog()" in the max_sessions loop.

If you move mdsc->mutex acquisition and max_sessions assignment into "if (req1 || req2)" branch, it can be trivially dropped before the loop.

Thanks,

Ilya

author
#4 lore raw
Quoted 131 lines · Ilya Dryomov
Quoted 130 lines · Xiubo Li
Quoted 114 lines · Ilya Dryomov
Quoted 102 lines · On Tue, Nov 8, 2022 at 2:56 PM <[email protected]> wrote:

From: Xiubo Li <[email protected]>

The request's r_session maybe changed when it was forwarded or resent.

Cc: [email protected]
URL: https://bugzilla.redhat.com/show_bug.cgi?id=2137955

---
   fs/ceph/caps.c | 60 ++++++++++++++++----------------------------------
   1 file changed, 19 insertions(+), 41 deletions(-)
fs/ceph/caps.c +9 -36
index 894adfb4a092..83f9e18e3169 100644
@@ -2297,8 +2297,10 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
          struct ceph_mds_client *mdsc = ceph_sb_to_client(inode->i_sb)->mdsc;
          struct ceph_inode_info *ci = ceph_inode(inode);
          struct ceph_mds_request *req1 = NULL, *req2 = NULL;
+       struct ceph_mds_session **sessions = NULL;
+       struct ceph_mds_session *s;
          unsigned int max_sessions;
-       int ret, err = 0;
+       int i, ret, err = 0;
 
          spin_lock(&ci->i_unsafe_lock);
          if (S_ISDIR(inode->i_mode) && !list_empty(&ci->i_unsafe_dirops)) {
@@ -2315,28 +2317,19 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
          }
          spin_unlock(&ci->i_unsafe_lock);
 
-       /*
-        * The mdsc->max_sessions is unlikely to be changed
-        * mostly, here we will retry it by reallocating the
-        * sessions array memory to get rid of the mdsc->mutex
-        * lock.
-        */
-retry:
-       max_sessions = mdsc->max_sessions;
-
          /*
           * Trigger to flush the journal logs in all the relevant MDSes
           * manually, or in the worst case we must wait at most 5 seconds
           * to wait the journal logs to be flushed by the MDSes periodically.
           */
-       if ((req1 || req2) && likely(max_sessions)) {
-               struct ceph_mds_session **sessions = NULL;
-               struct ceph_mds_session *s;
+       mutex_lock(&mdsc->mutex);
+       max_sessions = mdsc->max_sessions;
+       if (req1 || req2) {
                  struct ceph_mds_request *req;
-               int i;
 
                  sessions = kcalloc(max_sessions, sizeof(s), GFP_KERNEL);
                  if (!sessions) {
+                       mutex_unlock(&mdsc->mutex);
                          err = -ENOMEM;
                          goto out;
                  }
@@ -2346,18 +2339,8 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
                          list_for_each_entry(req, &ci->i_unsafe_dirops,
                                              r_unsafe_dir_item) {
                                  s = req->r_session;
-                               if (!s)
+                               if (!s || unlikely(s->s_mds >= max_sessions))
                                          continue;
-                               if (unlikely(s->s_mds >= max_sessions)) {
-                                       spin_unlock(&ci->i_unsafe_lock);
-                                       for (i = 0; i < max_sessions; i++) {
-                                               s = sessions[i];
-                                               if (s)
-                                                       ceph_put_mds_session(s);
-                                       }
-                                       kfree(sessions);
-                                       goto retry;
-                               }
                                  if (!sessions[s->s_mds]) {
                                          s = ceph_get_mds_session(s);
                                          sessions[s->s_mds] = s;
@@ -2368,18 +2351,8 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
                          list_for_each_entry(req, &ci->i_unsafe_iops,
                                              r_unsafe_target_item) {
                                  s = req->r_session;
-                               if (!s)
+                               if (!s || unlikely(s->s_mds >= max_sessions))
                                          continue;
-                               if (unlikely(s->s_mds >= max_sessions)) {
-                                       spin_unlock(&ci->i_unsafe_lock);
-                                       for (i = 0; i < max_sessions; i++) {
-                                               s = sessions[i];
-                                               if (s)
-                                                       ceph_put_mds_session(s);
-                                       }
-                                       kfree(sessions);
-                                       goto retry;
-                               }
                                  if (!sessions[s->s_mds]) {
                                          s = ceph_get_mds_session(s);
                                          sessions[s->s_mds] = s;

fs/ceph/caps.c static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode) · line 2369

Quoted 9 lines
fs/ceph/caps.c +3 -3
@@ -2391,13 +2364,18 @@ static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode)
                  /* the auth MDS */
                  spin_lock(&ci->i_ceph_lock);
                  if (ci->i_auth_cap) {
-                     s = ci->i_auth_cap->session;
-                     if (!sessions[s->s_mds])
-                             sessions[s->s_mds] = ceph_get_mds_session(s);
+                       s = ci->i_auth_cap->session;
+                       if (likely(s->s_mds < max_sessions)
+                           && !sessions[s->s_mds]) {

Hi Xiubo,

Quoted 1 line

Nit: keep && on the previous line for style consistency.

Sure. Will fix it.

Quoted 10 lines
Quoted 10 lines
fs/ceph/caps.c +6 -1
+                               sessions[s->s_mds] = ceph_get_mds_session(s);
+                       }
                  }
                  spin_unlock(&ci->i_ceph_lock);
+       }
+       mutex_unlock(&mdsc->mutex);
 
-               /* send flush mdlog request to MDSes */
+       /* send flush mdlog request to MDSes */
+       if (sessions) {
Quoted 7 lines
Quoted 4 lines
Quoted 2 lines

Since mdlog is flushed only in "if (req1 || req2)" case, why not keep max_sessions loop there and avoid sessions != NULL check?

This is because I must drop the mdsc->mutex before calling "send_flush_mdlog()" in the max_sessions loop.

If you move mdsc->mutex acquisition and max_sessions assignment into "if (req1 || req2)" branch, it can be trivially dropped before the loop.

Okay, sounds good.

Thanks!

- Xiubo

Quoted 4 lines

Thanks,

Ilya