Skip to contentSkip to conversation

ceph: don't leak snap_rwsem in handle_cap_grant

PATCH In review Jeff Layton 3 participants 8 messages Jun 3, 2022 – Jun 6, 2022 Ceph client

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 ceph: don't leak snap_rwsem in handle_cap_grant

When handle_cap_grant is called on an IMPORT op, then the snap_rwsem is held and the function is expected to release it before returning. It currently fails to do that in all cases which could lead to a deadlock.

URL: https://tracker.ceph.com/issues/55857

---
 fs/ceph/caps.c | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)
fs/ceph/caps.c +13 -14
index 258093e9074d..0a48bf829671 100644
@@ -3579,24 +3579,23 @@ static void handle_cap_grant(struct inode *inode,
 			fill_inline = true;
 	}
 
-	if (ci->i_auth_cap == cap &&
-	    le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
-		if (newcaps & ~extra_info->issued)
-			wake = true;
+	if (le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
+		if (ci->i_auth_cap == cap) {
+			if (newcaps & ~extra_info->issued)
+				wake = true;
+
+			if (ci->i_requested_max_size > max_size ||
+			    !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
+				/* re-request max_size if necessary */
+				ci->i_requested_max_size = 0;
+				wake = true;
+			}
 
-		if (ci->i_requested_max_size > max_size ||
-		    !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
-			/* re-request max_size if necessary */
-			ci->i_requested_max_size = 0;
-			wake = true;
+			ceph_kick_flushing_inode_caps(session, ci);
 		}
-
-		ceph_kick_flushing_inode_caps(session, ci);
-		spin_unlock(&ci->i_ceph_lock);
 		up_read(&session->s_mdsc->snap_rwsem);
-	} else {
-		spin_unlock(&ci->i_ceph_lock);
 	}
+	spin_unlock(&ci->i_ceph_lock);
 
 	if (fill_inline)
 		ceph_fill_inline_data(inode, NULL, extra_info->inline_data,
Signature · 1 line
2.36.1
#1 lore raw
Quoted 42 lines · Jeff Layton

When handle_cap_grant is called on an IMPORT op, then the snap_rwsem is held and the function is expected to release it before returning. It currently fails to do that in all cases which could lead to a deadlock.

URL: https://tracker.ceph.com/issues/55857

---
  fs/ceph/caps.c | 27 +++++++++++++--------------
  1 file changed, 13 insertions(+), 14 deletions(-)
fs/ceph/caps.c +12 -10
index 258093e9074d..0a48bf829671 100644
@@ -3579,24 +3579,23 @@ static void handle_cap_grant(struct inode *inode,
  			fill_inline = true;
  	}
 
-	if (ci->i_auth_cap == cap &&
-	    le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
-		if (newcaps & ~extra_info->issued)
-			wake = true;
+	if (le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
+		if (ci->i_auth_cap == cap) {
+			if (newcaps & ~extra_info->issued)
+				wake = true;
+
+			if (ci->i_requested_max_size > max_size ||
+			    !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
+				/* re-request max_size if necessary */
+				ci->i_requested_max_size = 0;
+				wake = true;
+			}
 
-		if (ci->i_requested_max_size > max_size ||
-		    !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
-			/* re-request max_size if necessary */
-			ci->i_requested_max_size = 0;
-			wake = true;
+			ceph_kick_flushing_inode_caps(session, ci);
  		}
-

fs/ceph/caps.c static void handle_cap_grant(struct inode *inode, · line 3601

Quoted 10 lines
fs/ceph/caps.c +1 -4
-		ceph_kick_flushing_inode_caps(session, ci);
-		spin_unlock(&ci->i_ceph_lock);
  		up_read(&session->s_mdsc->snap_rwsem);
-	} else {
-		spin_unlock(&ci->i_ceph_lock);
  	}
+	spin_unlock(&ci->i_ceph_lock);
 
  	if (fill_inline)
  		ceph_fill_inline_data(inode, NULL, extra_info->inline_data,

I am not sure what the following code in Line#4139 wants to do, more detail please see [1].

4131         if (msg_version >= 3) {
4132                 if (op == CEPH_CAP_OP_IMPORT) { 4133                         if (p + sizeof(*peer) > end) 4134                                 goto bad; 4135                         peer = p;
4136                         p += sizeof(*peer); 4137                 } else if (op == CEPH_CAP_OP_EXPORT) { 4138                         /* recorded in unused fields */ 4139                         peer = (void *)&h->size; 4140 }
4141         }

For the export case the members in the 'peer' are always assigned to 'random' values. And seems could cause this issue.

[1] https://github.com/ceph/ceph-client/blob/for-linus/fs/ceph/caps.c#L4134

I am still reading the code. Any idea ?

-- Xiubo

#2 lore raw
Quoted 69 lines · Xiubo Li
Quoted 42 lines · Jeff Layton

When handle_cap_grant is called on an IMPORT op, then the snap_rwsem is held and the function is expected to release it before returning. It currently fails to do that in all cases which could lead to a deadlock.

URL: https://tracker.ceph.com/issues/55857

---
  fs/ceph/caps.c | 27 +++++++++++++--------------
  1 file changed, 13 insertions(+), 14 deletions(-)
fs/ceph/caps.c +12 -10
index 258093e9074d..0a48bf829671 100644
@@ -3579,24 +3579,23 @@ static void handle_cap_grant(struct inode *inode,
  			fill_inline = true;
  	}
 
-	if (ci->i_auth_cap == cap &&
-	    le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
-		if (newcaps & ~extra_info->issued)
-			wake = true;
+	if (le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
+		if (ci->i_auth_cap == cap) {
+			if (newcaps & ~extra_info->issued)
+				wake = true;
+
+			if (ci->i_requested_max_size > max_size ||
+			    !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
+				/* re-request max_size if necessary */
+				ci->i_requested_max_size = 0;
+				wake = true;
+			}
 
-		if (ci->i_requested_max_size > max_size ||
-		    !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
-			/* re-request max_size if necessary */
-			ci->i_requested_max_size = 0;
-			wake = true;
+			ceph_kick_flushing_inode_caps(session, ci);
  		}
-

fs/ceph/caps.c static void handle_cap_grant(struct inode *inode, · line 3601

Quoted 10 lines
fs/ceph/caps.c +1 -4
-		ceph_kick_flushing_inode_caps(session, ci);
-		spin_unlock(&ci->i_ceph_lock);
  		up_read(&session->s_mdsc->snap_rwsem);
-	} else {
-		spin_unlock(&ci->i_ceph_lock);
  	}
+	spin_unlock(&ci->i_ceph_lock);
 
  	if (fill_inline)
  		ceph_fill_inline_data(inode, NULL, extra_info->inline_data,

I am not sure what the following code in Line#4139 wants to do, more detail please see [1].

4131         if (msg_version >= 3) {
4132                 if (op == CEPH_CAP_OP_IMPORT) { 4133                         if (p + sizeof(*peer) > end) 4134                                 goto bad; 4135                         peer = p;
4136                         p += sizeof(*peer); 4137                 } else if (op == CEPH_CAP_OP_EXPORT) { 4138                         /* recorded in unused fields */ 4139                         peer = (void *)&h->size; 4140 }
4141         }

Quoted 8 lines

For the export case the members in the 'peer' are always assigned to 'random' values. And seems could cause this issue.

[1] https://github.com/ceph/ceph-client/blob/for-linus/fs/ceph/caps.c#L4134

I am still reading the code. Any idea ?

I'm not sure that this is the case. It looks like most of the places in the mds code that make a CEPH_CAP_OP_EXPORT message call set_cap_peer just afterward. Why do you think this is given random values?

Signature · 1 line
Jeff Layton <[email protected]>
#3 lore raw
Quoted 72 lines · Jeff Layton
Quoted 71 lines · Xiubo Li
Quoted 42 lines · Jeff Layton

When handle_cap_grant is called on an IMPORT op, then the snap_rwsem is held and the function is expected to release it before returning. It currently fails to do that in all cases which could lead to a deadlock.

URL: https://tracker.ceph.com/issues/55857

---
   fs/ceph/caps.c | 27 +++++++++++++--------------
   1 file changed, 13 insertions(+), 14 deletions(-)
fs/ceph/caps.c +12 -10
index 258093e9074d..0a48bf829671 100644
@@ -3579,24 +3579,23 @@ static void handle_cap_grant(struct inode *inode,
   			fill_inline = true;
   	}
 
-	if (ci->i_auth_cap == cap &&
-	    le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
-		if (newcaps & ~extra_info->issued)
-			wake = true;
+	if (le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
+		if (ci->i_auth_cap == cap) {
+			if (newcaps & ~extra_info->issued)
+				wake = true;
+
+			if (ci->i_requested_max_size > max_size ||
+			    !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
+				/* re-request max_size if necessary */
+				ci->i_requested_max_size = 0;
+				wake = true;
+			}
 
-		if (ci->i_requested_max_size > max_size ||
-		    !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
-			/* re-request max_size if necessary */
-			ci->i_requested_max_size = 0;
-			wake = true;
+			ceph_kick_flushing_inode_caps(session, ci);
   		}
-

fs/ceph/caps.c static void handle_cap_grant(struct inode *inode, · line 3601

Quoted 10 lines
fs/ceph/caps.c +1 -4
-		ceph_kick_flushing_inode_caps(session, ci);
-		spin_unlock(&ci->i_ceph_lock);
   		up_read(&session->s_mdsc->snap_rwsem);
-	} else {
-		spin_unlock(&ci->i_ceph_lock);
   	}
+	spin_unlock(&ci->i_ceph_lock);
 
   	if (fill_inline)
   		ceph_fill_inline_data(inode, NULL, extra_info->inline_data,

I am not sure what the following code in Line#4139 wants to do, more detail please see [1].

4131         if (msg_version >= 3) {
4132                 if (op == CEPH_CAP_OP_IMPORT) { 4133                         if (p + sizeof(*peer) > end) 4134                                 goto bad; 4135                         peer = p;
4136                         p += sizeof(*peer); 4137                 } else if (op == CEPH_CAP_OP_EXPORT) { 4138                         /* recorded in unused fields */ 4139                         peer = (void *)&h->size; 4140 }
4141         }

For the export case the members in the 'peer' are always assigned to 'random' values. And seems could cause this issue.

Quoted 9 lines
Quoted 5 lines

[1] https://github.com/ceph/ceph-client/blob/for-linus/fs/ceph/caps.c#L4134

I am still reading the code. Any idea ?

I'm not sure that this is the case. It looks like most of the places in the mds code that make a CEPH_CAP_OP_EXPORT message call set_cap_peer just afterward. Why do you think this is given random values?

Misread the ceph message protocol related code. Please ignore this.

-- Xiubo

#4 lore raw
In reply to Jeff Layton
Quoted 42 lines · Jeff Layton

When handle_cap_grant is called on an IMPORT op, then the snap_rwsem is held and the function is expected to release it before returning. It currently fails to do that in all cases which could lead to a deadlock.

URL: https://tracker.ceph.com/issues/55857

---
  fs/ceph/caps.c | 27 +++++++++++++--------------
  1 file changed, 13 insertions(+), 14 deletions(-)
fs/ceph/caps.c +12 -10
index 258093e9074d..0a48bf829671 100644
@@ -3579,24 +3579,23 @@ static void handle_cap_grant(struct inode *inode,
  			fill_inline = true;
  	}
 
-	if (ci->i_auth_cap == cap &&
-	    le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
-		if (newcaps & ~extra_info->issued)
-			wake = true;
+	if (le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
+		if (ci->i_auth_cap == cap) {
+			if (newcaps & ~extra_info->issued)
+				wake = true;
+
+			if (ci->i_requested_max_size > max_size ||
+			    !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
+				/* re-request max_size if necessary */
+				ci->i_requested_max_size = 0;
+				wake = true;
+			}
 
-		if (ci->i_requested_max_size > max_size ||
-		    !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
-			/* re-request max_size if necessary */
-			ci->i_requested_max_size = 0;
-			wake = true;
+			ceph_kick_flushing_inode_caps(session, ci);
  		}
-

fs/ceph/caps.c static void handle_cap_grant(struct inode *inode, · line 3601

Quoted 10 lines
fs/ceph/caps.c +1 -4
-		ceph_kick_flushing_inode_caps(session, ci);
-		spin_unlock(&ci->i_ceph_lock);
  		up_read(&session->s_mdsc->snap_rwsem);
-	} else {
-		spin_unlock(&ci->i_ceph_lock);
  	}
+	spin_unlock(&ci->i_ceph_lock);
 
  	if (fill_inline)
  		ceph_fill_inline_data(inode, NULL, extra_info->inline_data,

Anyhow the snap_rwsem should be released.

Merged into testing branch. Thanks Jeff.

-- Xiubo

Reviewed-by on v1 1/1
#5 lore raw
In reply to Jeff Layton
Quoted 5 lines · Jeff Layton

When handle_cap_grant is called on an IMPORT op, then the snap_rwsem is held and the function is expected to release it before returning. It currently fails to do that in all cases which could lead to a deadlock.

URL: https://tracker.ceph.com/issues/55857

This looks good. Maybe it could have here a 'Fixes: e8a4d26771547'. Otherwise:

Cheers,

Signature · 1 line
Luís
Quoted 51 lines
---
 fs/ceph/caps.c | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)
fs/ceph/caps.c +13 -14
index 258093e9074d..0a48bf829671 100644
@@ -3579,24 +3579,23 @@ static void handle_cap_grant(struct inode *inode,
 			fill_inline = true;
 	}
 
-	if (ci->i_auth_cap == cap &&
-	    le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
-		if (newcaps & ~extra_info->issued)
-			wake = true;
+	if (le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
+		if (ci->i_auth_cap == cap) {
+			if (newcaps & ~extra_info->issued)
+				wake = true;
+
+			if (ci->i_requested_max_size > max_size ||
+			    !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
+				/* re-request max_size if necessary */
+				ci->i_requested_max_size = 0;
+				wake = true;
+			}
 
-		if (ci->i_requested_max_size > max_size ||
-		    !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
-			/* re-request max_size if necessary */
-			ci->i_requested_max_size = 0;
-			wake = true;
+			ceph_kick_flushing_inode_caps(session, ci);
 		}
-
-		ceph_kick_flushing_inode_caps(session, ci);
-		spin_unlock(&ci->i_ceph_lock);
 		up_read(&session->s_mdsc->snap_rwsem);
-	} else {
-		spin_unlock(&ci->i_ceph_lock);
 	}
+	spin_unlock(&ci->i_ceph_lock);
 
 	if (fill_inline)
 		ceph_fill_inline_data(inode, NULL, extra_info->inline_data,
Signature · 1 line
2.36.1
#6 lore raw
Quoted 6 lines · Luís Henriques
Quoted 4 lines · Jeff Layton

When handle_cap_grant is called on an IMPORT op, then the snap_rwsem is held and the function is expected to release it before returning. It currently fails to do that in all cases which could lead to a deadlock.

Quoted 8 lines
Quoted 1 line

URL: https://tracker.ceph.com/issues/55857

This looks good. Maybe it could have here a 'Fixes: e8a4d26771547'. Otherwise:

Cheers,

Thanks,

Actually, I think this got broken in 6f05b30ea063a2 (ceph: reset i_requested_max_size if file write is not wanted).

The problem is the && part of the condition. We need to release the rwsem regardless of whether ci->i_auth_cap == cap.

Signature · 1 line
Jeff Layton <[email protected]>
#7 lore raw
Quoted 11 lines · Jeff Layton
Quoted 10 lines · Luís Henriques
Quoted 5 lines · Jeff Layton

When handle_cap_grant is called on an IMPORT op, then the snap_rwsem is held and the function is expected to release it before returning. It currently fails to do that in all cases which could lead to a deadlock.

URL: https://tracker.ceph.com/issues/55857

This looks good. Maybe it could have here a 'Fixes: e8a4d26771547'. Otherwise:

Quoted 7 lines
Quoted 3 lines

Cheers,

Thanks,

Actually, I think this got broken in 6f05b30ea063a2 (ceph: reset i_requested_max_size if file write is not wanted).

Have updated it.

-- Xiubo

Quoted 2 lines

The problem is the && part of the condition. We need to release the rwsem regardless of whether ci->i_auth_cap == cap.