Skip to contentSkip to conversation

ceph: wake waiters on any IMPORT that grants new caps

PATCH In review Jeff Layton 2 participants 4 messages Jan 27, 2022 – Feb 2, 2022 1 message not in the archive Ceph client

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 ceph: wake waiters on any IMPORT that grants new caps

I've noticed an intermittent hang waiting for caps in some testing. What I see is that the client will try to get caps for an operation (e.g. a read), and ends up waiting on the waitqueue forever. The caps debugfs file however shows that the caps it's waiting on have already been granted.

The current grant handling code will wake the waitqueue when it sees that there are newly-granted caps in the issued set. On an import however, we'll end up adding a new cap first, which fools the logic into thinking that nothing has changed. A later hack in the code works around this, but only for auth caps.

Ensure we wake the waiters whenever we get an IMPORT that grants new caps for the inode.

URL: https://tracker.ceph.com/issues/54044

---
 fs/ceph/caps.c | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

I'm still testing this patch, but I think this may be the cause of some mysterious hangs I've hit in testing.

fs/ceph/caps.c +12 -11
index e668cdb9c99e..06b65a68e920 100644
@@ -3541,21 +3541,22 @@ static void handle_cap_grant(struct inode *inode,
 			fill_inline = true;
 	}
 
-	if (ci->i_auth_cap == cap &&
-	    le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
+	if (le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
 		if (newcaps & ~extra_info->issued)
 			wake = true;
 
-		if (ci->i_requested_max_size > max_size ||
-		    !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
-			/* re-request max_size if necessary */
-			ci->i_requested_max_size = 0;
-			wake = true;
-		}
+		if (ci->i_auth_cap == cap) {
+			if (ci->i_requested_max_size > max_size ||
+			    !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
+				/* re-request max_size if necessary */
+				ci->i_requested_max_size = 0;
+				wake = true;
+			}
 
-		ceph_kick_flushing_inode_caps(session, ci);
-		spin_unlock(&ci->i_ceph_lock);
-		up_read(&session->s_mdsc->snap_rwsem);
+			ceph_kick_flushing_inode_caps(session, ci);
+			spin_unlock(&ci->i_ceph_lock);
+			up_read(&session->s_mdsc->snap_rwsem);
+		}
 	} else {
 		spin_unlock(&ci->i_ceph_lock);
 	}
Signature · 1 line
2.34.1
#1 lore raw
Quoted 7 lines · Jeff Layton

I've noticed an intermittent hang waiting for caps in some testing. What I see is that the client will try to get caps for an operation (e.g. a read), and ends up waiting on the waitqueue forever. The caps debugfs file however shows that the caps it's waiting on have already been granted.

Quoted 5 lines

The current grant handling code will wake the waitqueue when it sees that there are newly-granted caps in the issued set. On an import however, we'll end up adding a new cap first, which fools the logic into thinking that nothing has changed. A later hack in the code works around this, but only for auth caps.

not right. handle_cap_import() saves old issued to extra_info->issued.

Quoted 54 lines

Ensure we wake the waiters whenever we get an IMPORT that grants new caps for the inode.

URL: https://tracker.ceph.com/issues/54044

---
 fs/ceph/caps.c | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

I'm still testing this patch, but I think this may be the cause of some mysterious hangs I've hit in testing.

fs/ceph/caps.c +12 -12
index e668cdb9c99e..06b65a68e920 100644
@@ -3541,21 +3541,22 @@ static void handle_cap_grant(struct inode *inode,
                        fill_inline = true;
        }
 
-       if (ci->i_auth_cap == cap &&
-           le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
+       if (le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
                if (newcaps & ~extra_info->issued)
                        wake = true;
 
-               if (ci->i_requested_max_size > max_size ||
-                   !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
-                       /* re-request max_size if necessary */
-                       ci->i_requested_max_size = 0;
-                       wake = true;
-               }
+               if (ci->i_auth_cap == cap) {
+                       if (ci->i_requested_max_size > max_size ||
+                           !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
+                               /* re-request max_size if necessary */
+                               ci->i_requested_max_size = 0;
+                               wake = true;
+                       }
 
-               ceph_kick_flushing_inode_caps(session, ci);
-               spin_unlock(&ci->i_ceph_lock);
-               up_read(&session->s_mdsc->snap_rwsem);
+                       ceph_kick_flushing_inode_caps(session, ci);
+                       spin_unlock(&ci->i_ceph_lock);
+                       up_read(&session->s_mdsc->snap_rwsem);
+               }
        } else {
                spin_unlock(&ci->i_ceph_lock);
        }
--

2.34.1

#2 lore raw
Quoted 8 lines · Zheng
Quoted 7 lines · Jeff Layton

I've noticed an intermittent hang waiting for caps in some testing. What I see is that the client will try to get caps for an operation (e.g. a read), and ends up waiting on the waitqueue forever. The caps debugfs file however shows that the caps it's waiting on have already been granted.

Quoted 8 lines
Quoted 5 lines

The current grant handling code will wake the waitqueue when it sees that there are newly-granted caps in the issued set. On an import however, we'll end up adding a new cap first, which fools the logic into thinking that nothing has changed. A later hack in the code works around this, but only for auth caps.

not right. handle_cap_import() saves old issued to extra_info->issued.

It does save the old issued value to extra_info->issued, but handle_cap grant consults cap->issued for most of its logic. It does check against extra_info->issued for auth caps to determine whether to wake waiters, but doesn't do that for non-auth caps. This patch corrects that.

If you think this patch isn't right, can you elaborate on why and what would make it correct?

Thanks,
Jeff

Quoted 54 lines
Quoted 54 lines

Ensure we wake the waiters whenever we get an IMPORT that grants new caps for the inode.

URL: https://tracker.ceph.com/issues/54044

---
 fs/ceph/caps.c | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

I'm still testing this patch, but I think this may be the cause of some mysterious hangs I've hit in testing.

fs/ceph/caps.c +12 -12
index e668cdb9c99e..06b65a68e920 100644
@@ -3541,21 +3541,22 @@ static void handle_cap_grant(struct inode *inode,
                        fill_inline = true;
        }
 
-       if (ci->i_auth_cap == cap &&
-           le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
+       if (le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
                if (newcaps & ~extra_info->issued)
                        wake = true;
 
-               if (ci->i_requested_max_size > max_size ||
-                   !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
-                       /* re-request max_size if necessary */
-                       ci->i_requested_max_size = 0;
-                       wake = true;
-               }
+               if (ci->i_auth_cap == cap) {
+                       if (ci->i_requested_max_size > max_size ||
+                           !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
+                               /* re-request max_size if necessary */
+                               ci->i_requested_max_size = 0;
+                               wake = true;
+                       }
 
-               ceph_kick_flushing_inode_caps(session, ci);
-               spin_unlock(&ci->i_ceph_lock);
-               up_read(&session->s_mdsc->snap_rwsem);
+                       ceph_kick_flushing_inode_caps(session, ci);
+                       spin_unlock(&ci->i_ceph_lock);
+                       up_read(&session->s_mdsc->snap_rwsem);
+               }
        } else {
                spin_unlock(&ci->i_ceph_lock);
        }
--

2.34.1

Signature · 1 line
Jeff Layton <[email protected]>
#4 lore raw
In reply to a message not in the archive
Quoted 41 lines · Zheng
Quoted 28 lines · Jeff Layton <[email protected]>于2022年1月29日 周六18:36写道:
Quoted 23 lines · Zheng
Quoted 11 lines · Jeff Layton

I've noticed an intermittent hang waiting for caps in some testing. What
I see is that the client will try to get caps for an operation (e.g. a
read), and ends up waiting on the waitqueue forever. The caps debugfs
file however shows that the caps it's waiting on have already been
granted.

Quoted 9 lines

The current grant handling code will wake the waitqueue when it sees
that there are newly-granted caps in the issued set. On an import
however, we'll end up adding a new cap first, which fools the logic into
thinking that nothing has changed. A later hack in the code works around
this, but only for auth caps.

Quoted 3 lines

not right. handle_cap_import() saves old issued to extra_info-

Quoted 1 line

issued.

Quoted 8 lines

It does save the old issued value to extra_info->issued, but handle_cap
grant consults cap->issued for most of its logic. It does check against
extra_info->issued for auth caps to determine whether to wake waiters,
but doesn't do that for non-auth caps. This patch corrects that.

Quoted 5 lines

non auth mds never imports caps. ‘ci->i_auth_cap != cap’ happened only when client receives out-of-order messages for successive cap import/export. It’s unlikely in your case because you saw caps were there.

Ok! I'll drop this patch, and keep hunting for how we're not getting awoken. Let me know if you see any gaps in how "wake" gets set.

Thanks,
Jeff

Quoted 72 lines
Quoted 72 lines

If you think this patch isn't right, can you elaborate on why and what
would make it correct?

Thanks,
Jeff

Quoted 62 lines
Quoted 62 lines

Ensure we wake the waiters whenever we get an IMPORT that grants new
caps for the inode.

URL: https://tracker.ceph.com/issues/54044

---
  fs/ceph/caps.c | 23 ++++++++++++-----------

  1 file changed, 12 insertions(+), 11 deletions(-)

I'm still testing this patch, but I think this may be the cause of some
mysterious hangs I've hit in testing.

fs/ceph/caps.c +3 -8
index e668cdb9c99e..06b65a68e920 100644
@@ -3541,21 +3541,22 @@ static void handle_cap_grant(struct
 inode *inode,
                          fill_inline = true;
          }
 
-       if (ci->i_auth_cap == cap &&
-           le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
+       if (le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) {
                  if (newcaps & ~extra_info->issued)
                          wake = true;
 
-               if (ci->i_requested_max_size > max_size ||
-                   !(le32_to_cpu(grant->wanted) &
 CEPH_CAP_ANY_FILE_WR)) {
-                       /* re-request max_size if necessary */
-                       ci->i_requested_max_size = 0;
-                       wake = true;
-               }
+               if (ci->i_auth_cap == cap) {
+                       if (ci->i_requested_max_size > max_size

||
+                           !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) {
+                               /* re-request max_size if necessary */
+                               ci->i_requested_max_size = 0; +                               wake = true;
+                       }

-               ceph_kick_flushing_inode_caps(session, ci); -               spin_unlock(&ci->i_ceph_lock);
-               up_read(&session->s_mdsc->snap_rwsem); +                       ceph_kick_flushing_inode_caps(session, ci);
+                       spin_unlock(&ci->i_ceph_lock); +                       up_read(&session->s_mdsc->snap_rwsem); +               }
         } else {
                 spin_unlock(&ci->i_ceph_lock);
         }
--
2.34.1

Signature · 1 line
Jeff Layton <[email protected]>