Skip to contentSkip to conversation

ceph: force a cap message when a deferred revoke can't be acked immediately

PATCH Superseded Max Kellermann 4 participants 4 messages Aug 12 – Aug 17

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 ceph: force a cap message when a deferred revoke can't be acked immediately

When the MDS revokes capabilities, handle_cap_grant() normally guarantees a response by setting CHECK_CAPS_FLUSH_FORCE (see commit 31634d7597d8 ("ceph: force sending a cap update msg back to MDS for revoke op")), so ceph_check_caps() sends a cap message even if the client would otherwise decide it has nothing to do. That guarantee is skipped whenever the revoke has to be deferred (via revoke_wait):
revoking Fb while dirty data is still buffered (writeback is queued first) or revoking Fc while pages are cached (async invalidation is queued first).

In those cases, the ack is left to the deferred completion (ceph_put_wrbuffer_cap_refs() after writeback, or the invalidate worker after invalidation); both of which call ceph_check_caps(ci,0) i.e. without CHECK_CAPS_FLUSH_FORCE. Nothing gets sent under one of the following conditions:

- the inode is retaining caps because the file was used recently (file_wanted != 0; retain |= CEPH_CAP_ANY)

- the revoked cap is still used because the page was re-cached (e.g. a file being re-read)

- the MDS has meanwhile re-granted, so issued==implemented and the client sees nothing being revoked

The client then never emits the cap message which the MDS is waiting for. The MDS blocks on the revoke indefinitely and logs, for minutes or hours:

client.NNN isn't responding to mclientcaps(revoke), ino 0x... pending pAsxLsXsxFsxcrwb issued pAsxLsXsxFsxcrwb, sent 964.899182 seconds ago

The client-side state at that point shows the full cap set still issued, nothing in the revoking/flushing sets. Thus nothing gets sent.

This patch fixes it by remembering that a forced response is expected. When a revoke is deferred, set CEPH_I_FLUSH_FORCE on the inode. ceph_check_caps() replays it as CHECK_CAPS_FLUSH_FORCE, so whichever path re-checks the inode next (the writeback/invalidate completion, the delayed worker, or any other caller) is guaranteed to send a cap message to the MDS. __prep_cap() clears the flag once a message is actually built.

This is the deferred-path counterpart of the existing CHECK_CAPS_FLUSH_FORCE handling; a normal (non-deferred) revoke still forces the response inline as before.

Fixes 31634d7597d8 ("ceph: force sending a cap update msg back to MDS for revoke op")
Fixes 257e6172ab36 ("ceph: don't let check_caps skip sending responses for revoke msgs")

---
v1->v2: fix CEPH_I_FLUSH_FORCE/CEPH_I_FLUSH_FORCE_BIT mixup

---
 fs/ceph/caps.c  | 40 +++++++++++++++++++++++++++++++++-------
 fs/ceph/super.h |  5 +++++
 2 files changed, 38 insertions(+), 7 deletions(-)
fs/ceph/caps.c +33 -7
index d7283fb54cec..8ad17787b172 100644
@@ -1410,6 +1410,7 @@ static void __prep_cap(struct cap_msg_args *arg, struct ceph_cap *cap,
 	BUG_ON((retain & CEPH_CAP_PIN) == 0);
 
 	clear_bit(CEPH_I_FLUSH_BIT, &ci->i_ceph_flags);
+	clear_bit(CEPH_I_FLUSH_FORCE_BIT, &ci->i_ceph_flags);
 
 	cap->issued &= retain;  /* drop bits we don't want */
 	/*
@@ -2038,6 +2039,14 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags)
 
 	if (ci->i_ceph_flags & CEPH_I_FLUSH)
 		flags |= CHECK_CAPS_FLUSH;
+	/*
+	 * A revoke whose response was deferred (see handle_cap_grant()) must
+	 * still be acknowledged.  Replay the forced flush here so that even a
+	 * check triggered by writeback/invalidation completion sends a cap
+	 * message to the MDS.
+	 */
+	if (ci->i_ceph_flags & CEPH_I_FLUSH_FORCE)
+		flags |= CHECK_CAPS_FLUSH_FORCE;
 retry:
 	/* Caps wanted by virtue of active open files. */
 	file_wanted = __ceph_caps_file_wanted(ci);
@@ -3757,13 +3766,30 @@ static void handle_cap_grant(struct inode *inode,
 	BUG_ON(cap->issued & ~cap->implemented);
 
 	/* don't let check_caps skip sending a response to MDS for revoke msgs */
-	if (!revoke_wait && le32_to_cpu(grant->op) == CEPH_CAP_OP_REVOKE) {
-		cap->mds_wanted = 0;
-		flags |= CHECK_CAPS_FLUSH_FORCE;
-		if (cap == ci->i_auth_cap)
-			check_caps = 1; /* check auth cap only */
-		else
-			check_caps = 2; /* check all caps */
+	if (le32_to_cpu(grant->op) == CEPH_CAP_OP_REVOKE) {
+		if (revoke_wait) {
+			/*
+			 * We can't ack the revoke yet: the response is deferred
+			 * until the writeback or cache invalidation queued above
+			 * completes.  Set the CEPH_I_FLUSH_FORCE flag to remember
+			 * that a forced cap message is owed so that deferred
+			 * completion (ceph_put_wrbuffer_cap_refs() or the
+			 * invalidate worker, both of which call ceph_check_caps())
+			 * actually sends one, even if by then the revoked caps look
+			 * unused, the inode is retaining caps, or the MDS has
+			 * re-granted them.  Without this, the cap message is never
+			 * sent and the MDS hangs ("isn't responding to
+			 * mclientcaps(revoke)").
+			 */
+			ci->i_ceph_flags |= CEPH_I_FLUSH_FORCE;
+		} else {
+			cap->mds_wanted = 0;
+			flags |= CHECK_CAPS_FLUSH_FORCE;
+			if (cap == ci->i_auth_cap)
+				check_caps = 1; /* check auth cap only */
+			else
+				check_caps = 2; /* check all caps */
+		}
 	}
 
 	if (extra_info->inline_version > 0 &&
fs/ceph/super.h +5 -0
index 1d6aab060780..878440a2eb3b 100644
@@ -687,6 +687,10 @@ static inline struct inode *ceph_find_inode(struct super_block *sb,
 #define CEPH_I_ASYNC_CREATE_BIT		(12) /* async create in flight for this */
 #define CEPH_I_SHUTDOWN_BIT		(13) /* inode is no longer usable */
 #define CEPH_I_ASYNC_CHECK_CAPS_BIT	(14) /* check caps after async creating finishes */
+#define CEPH_I_FLUSH_FORCE_BIT		(15) /* a revoke's response was deferred;
+					      * force a cap message to the MDS once
+					      * the deferred work completes
+					      */
 
 #define CEPH_I_DIR_ORDERED		(1 << CEPH_I_DIR_ORDERED_BIT)
 #define CEPH_I_FLUSH			(1 << CEPH_I_FLUSH_BIT)
@@ -699,6 +703,7 @@ static inline struct inode *ceph_find_inode(struct super_block *sb,
 #define CEPH_I_ODIRECT			(1 << CEPH_I_ODIRECT_BIT)
 #define CEPH_I_ASYNC_CREATE		(1 << CEPH_I_ASYNC_CREATE_BIT)
 #define CEPH_I_SHUTDOWN			(1 << CEPH_I_SHUTDOWN_BIT)
+#define CEPH_I_FLUSH_FORCE		(1 << CEPH_I_FLUSH_FORCE_BIT)
 
 /*
  * Masks of ceph inode work.
Signature · 1 line
2.47.3
#1 lore raw
Quoted 73 lines · Max Kellermann

When the MDS revokes capabilities, handle_cap_grant() normally guarantees a response by setting CHECK_CAPS_FLUSH_FORCE (see commit 31634d7597d8 ("ceph: force sending a cap update msg back to MDS for revoke op")), so ceph_check_caps() sends a cap message even if the client would otherwise decide it has nothing to do. That guarantee is skipped whenever the revoke has to be deferred (via revoke_wait):
revoking Fb while dirty data is still buffered (writeback is queued first) or revoking Fc while pages are cached (async invalidation is queued first).

In those cases, the ack is left to the deferred completion (ceph_put_wrbuffer_cap_refs() after writeback, or the invalidate worker after invalidation); both of which call ceph_check_caps(ci,0) i.e. without CHECK_CAPS_FLUSH_FORCE. Nothing gets sent under one of the following conditions:

- the inode is retaining caps because the file was used recently (file_wanted != 0; retain |= CEPH_CAP_ANY)

- the revoked cap is still used because the page was re-cached (e.g. a file being re-read)

- the MDS has meanwhile re-granted, so issued==implemented and the client sees nothing being revoked

The client then never emits the cap message which the MDS is waiting for. The MDS blocks on the revoke indefinitely and logs, for minutes or hours:

client.NNN isn't responding to mclientcaps(revoke), ino 0x... pending pAsxLsXsxFsxcrwb issued pAsxLsXsxFsxcrwb, sent 964.899182 seconds ago

The client-side state at that point shows the full cap set still issued, nothing in the revoking/flushing sets. Thus nothing gets sent.

This patch fixes it by remembering that a forced response is expected. When a revoke is deferred, set CEPH_I_FLUSH_FORCE on the inode. ceph_check_caps() replays it as CHECK_CAPS_FLUSH_FORCE, so whichever path re-checks the inode next (the writeback/invalidate completion, the delayed worker, or any other caller) is guaranteed to send a cap message to the MDS. __prep_cap() clears the flag once a message is actually built.

This is the deferred-path counterpart of the existing CHECK_CAPS_FLUSH_FORCE handling; a normal (non-deferred) revoke still forces the response inline as before.

Fixes 31634d7597d8 ("ceph: force sending a cap update msg back to MDS for revoke op")
Fixes 257e6172ab36 ("ceph: don't let check_caps skip sending responses for revoke msgs")

---
v1->v2: fix CEPH_I_FLUSH_FORCE/CEPH_I_FLUSH_FORCE_BIT mixup

---
 fs/ceph/caps.c  | 40 +++++++++++++++++++++++++++++++++-------
 fs/ceph/super.h |  5 +++++
 2 files changed, 38 insertions(+), 7 deletions(-)
fs/ceph/caps.c +1 -0
index d7283fb54cec..8ad17787b172 100644
@@ -1410,6 +1410,7 @@ static void __prep_cap(struct cap_msg_args *arg, struct ceph_cap *cap,
        BUG_ON((retain & CEPH_CAP_PIN) == 0);
 
        clear_bit(CEPH_I_FLUSH_BIT, &ci->i_ceph_flags);
+       clear_bit(CEPH_I_FLUSH_FORCE_BIT, &ci->i_ceph_flags);
 
        cap->issued &= retain;  /* drop bits we don't want */
        /*

fs/ceph/caps.c void ceph_check_caps(struct ceph_inode_info *ci, int flags) · line 2048

Quoted 10 lines
fs/ceph/caps.c +7 -0
@@ -2038,6 +2039,14 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags)
 
        if (ci->i_ceph_flags & CEPH_I_FLUSH)
                flags |= CHECK_CAPS_FLUSH;
+       /*
+        * A revoke whose response was deferred (see handle_cap_grant()) must
+        * still be acknowledged.  Replay the forced flush here so that even a
+        * check triggered by writeback/invalidation completion sends a cap
+        * message to the MDS.
+        */
+       if (ci->i_ceph_flags & CEPH_I_FLUSH_FORCE)

Since CEPH_I_FLUSH_FORCE is now defined as a bit mask and the flag is set/cleared with set_bit()/clear_bit(), could we use test_bit() here as well?

if (test_bit(CEPH_I_FLUSH_FORCE_BIT, &ci->i_ceph_flags))

        flags |= CHECK_CAPS_FLUSH_FORCE;

This keeps the set/test/clear operations consistently using the _BIT definition and avoids mixing the bit index and mask forms.

Quoted 21 lines
fs/ceph/caps.c +7 -7
+               flags |= CHECK_CAPS_FLUSH_FORCE;
 retry:
        /* Caps wanted by virtue of active open files. */
        file_wanted = __ceph_caps_file_wanted(ci);
@@ -3757,13 +3766,30 @@ static void handle_cap_grant(struct inode *inode,
        BUG_ON(cap->issued & ~cap->implemented);
 
        /* don't let check_caps skip sending a response to MDS for revoke msgs */
-       if (!revoke_wait && le32_to_cpu(grant->op) == CEPH_CAP_OP_REVOKE) {
-               cap->mds_wanted = 0;
-               flags |= CHECK_CAPS_FLUSH_FORCE;
-               if (cap == ci->i_auth_cap)
-                       check_caps = 1; /* check auth cap only */
-               else
-                       check_caps = 2; /* check all caps */
+       if (le32_to_cpu(grant->op) == CEPH_CAP_OP_REVOKE) {
+               if (revoke_wait) {
+                       /*
+                        * We can't ack the revoke yet: the response is deferred
+                        * until the writeback or cache invalidation queued above
+                        * completes.  Set the CEPH_I_FLUSH_FORCE flag to remember

fs/ceph/caps.c static void handle_cap_grant(struct inode *inode, · line 3784

Quoted 10 lines
fs/ceph/caps.c +10 -0
+                        * that a forced cap message is owed so that deferred
+                        * completion (ceph_put_wrbuffer_cap_refs() or the
+                        * invalidate worker, both of which call ceph_check_caps())
+                        * actually sends one, even if by then the revoked caps look
+                        * unused, the inode is retaining caps, or the MDS has
+                        * re-granted them.  Without this, the cap message is never
+                        * sent and the MDS hangs ("isn't responding to
+                        * mclientcaps(revoke)").
+                        */
+                       ci->i_ceph_flags |= CEPH_I_FLUSH_FORCE;

Similar here?

Thanks
- Xiubo

Quoted 37 lines
fs/ceph/caps.c +8 -0
+               } else {
+                       cap->mds_wanted = 0;
+                       flags |= CHECK_CAPS_FLUSH_FORCE;
+                       if (cap == ci->i_auth_cap)
+                               check_caps = 1; /* check auth cap only */
+                       else
+                               check_caps = 2; /* check all caps */
+               }
        }
 
        if (extra_info->inline_version > 0 &&
fs/ceph/super.h +5 -1
index 1d6aab060780..878440a2eb3b 100644
@@ -687,6 +687,10 @@ static inline struct inode *ceph_find_inode(struct super_block *sb,
 #define CEPH_I_ASYNC_CREATE_BIT                (12) /* async create in flight for this */
 #define CEPH_I_SHUTDOWN_BIT            (13) /* inode is no longer usable */
 #define CEPH_I_ASYNC_CHECK_CAPS_BIT    (14) /* check caps after async creating finishes */
+#define CEPH_I_FLUSH_FORCE_BIT         (15) /* a revoke's response was deferred;
+                                             * force a cap message to the MDS once
+                                             * the deferred work completes
+                                             */
 
 #define CEPH_I_DIR_ORDERED             (1 << CEPH_I_DIR_ORDERED_BIT)
 #define CEPH_I_FLUSH                   (1 << CEPH_I_FLUSH_BIT)
@@ -699,6 +703,7 @@ static inline struct inode *ceph_find_inode(struct super_block *sb,
 #define CEPH_I_ODIRECT                 (1 << CEPH_I_ODIRECT_BIT)
 #define CEPH_I_ASYNC_CREATE            (1 << CEPH_I_ASYNC_CREATE_BIT)
 #define CEPH_I_SHUTDOWN                        (1 << CEPH_I_SHUTDOWN_BIT)
+#define CEPH_I_FLUSH_FORCE             (1 << CEPH_I_FLUSH_FORCE_BIT)
 
 /*
  * Masks of ceph inode work.
--

2.47.3

maintainer
#2 lore raw
In reply to Max Kellermann

Hi Max,

Two issues in v2:

The clear_bit(CEPH_I_FLUSH_FORCE_BIT) in ceph_check_caps() fires whenever any cap flush completes on the inode, not only when the deferred-revoke cap message itself is done. A concurrent cap flush for a different cap clears the flag prematurely, before the deferred revoke is handled.

Also, ci->i_ceph_flags |= CEPH_I_FLUSH_FORCE is a non-atomic read-modify-write — every other flag on i_ceph_flags uses set_bit() / clear_bit(). This one should be set_bit(CEPH_I_FLUSH_FORCE_BIT, &ci->i_ceph_flags).

Could you respin?

--
Alex Markuze

#3 lore raw
In reply to Max Kellermann
Quoted 37 lines · Max Kellermann

When the MDS revokes capabilities, handle_cap_grant() normally guarantees a response by setting CHECK_CAPS_FLUSH_FORCE (see commit 31634d7597d8 ("ceph: force sending a cap update msg back to MDS for revoke op")), so ceph_check_caps() sends a cap message even if the client would otherwise decide it has nothing to do. That guarantee is skipped whenever the revoke has to be deferred (via revoke_wait):
revoking Fb while dirty data is still buffered (writeback is queued first) or revoking Fc while pages are cached (async invalidation is queued first).

In those cases, the ack is left to the deferred completion (ceph_put_wrbuffer_cap_refs() after writeback, or the invalidate worker after invalidation); both of which call ceph_check_caps(ci,0) i.e. without CHECK_CAPS_FLUSH_FORCE. Nothing gets sent under one of the following conditions:

- the inode is retaining caps because the file was used recently (file_wanted != 0; retain |= CEPH_CAP_ANY)

- the revoked cap is still used because the page was re-cached (e.g. a file being re-read)

- the MDS has meanwhile re-granted, so issued==implemented and the client sees nothing being revoked

The client then never emits the cap message which the MDS is waiting for. The MDS blocks on the revoke indefinitely and logs, for minutes or hours:

client.NNN isn't responding to mclientcaps(revoke), ino 0x... pending pAsxLsXsxFsxcrwb issued pAsxLsXsxFsxcrwb, sent 964.899182 seconds ago

The client-side state at that point shows the full cap set still issued, nothing in the revoking/flushing sets. Thus nothing gets sent.

Quoted 7 lines

This patch fixes it by remembering that a forced response is expected. When a revoke is deferred, set CEPH_I_FLUSH_FORCE on the inode. ceph_check_caps() replays it as CHECK_CAPS_FLUSH_FORCE, so whichever path re-checks the inode next (the writeback/invalidate completion, the delayed worker, or any other caller) is guaranteed to send a cap message to the MDS. __prep_cap() clears the flag once a message is actually built.

Thanks for the patch, Max. This probably fixes
https://tracker.ceph.com/issues/75735

Quoted 108 lines

This is the deferred-path counterpart of the existing CHECK_CAPS_FLUSH_FORCE handling; a normal (non-deferred) revoke still forces the response inline as before.

Fixes 31634d7597d8 ("ceph: force sending a cap update msg back to MDS for revoke op")
Fixes 257e6172ab36 ("ceph: don't let check_caps skip sending responses for revoke msgs")

---
v1->v2: fix CEPH_I_FLUSH_FORCE/CEPH_I_FLUSH_FORCE_BIT mixup

---
 fs/ceph/caps.c  | 40 +++++++++++++++++++++++++++++++++-------
 fs/ceph/super.h |  5 +++++
 2 files changed, 38 insertions(+), 7 deletions(-)
fs/ceph/caps.c +33 -7
index d7283fb54cec..8ad17787b172 100644
@@ -1410,6 +1410,7 @@ static void __prep_cap(struct cap_msg_args *arg, struct ceph_cap *cap,
        BUG_ON((retain & CEPH_CAP_PIN) == 0);
 
        clear_bit(CEPH_I_FLUSH_BIT, &ci->i_ceph_flags);
+       clear_bit(CEPH_I_FLUSH_FORCE_BIT, &ci->i_ceph_flags);
 
        cap->issued &= retain;  /* drop bits we don't want */
        /*
@@ -2038,6 +2039,14 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags)
 
        if (ci->i_ceph_flags & CEPH_I_FLUSH)
                flags |= CHECK_CAPS_FLUSH;
+       /*
+        * A revoke whose response was deferred (see handle_cap_grant()) must
+        * still be acknowledged.  Replay the forced flush here so that even a
+        * check triggered by writeback/invalidation completion sends a cap
+        * message to the MDS.
+        */
+       if (ci->i_ceph_flags & CEPH_I_FLUSH_FORCE)
+               flags |= CHECK_CAPS_FLUSH_FORCE;
 retry:
        /* Caps wanted by virtue of active open files. */
        file_wanted = __ceph_caps_file_wanted(ci);
@@ -3757,13 +3766,30 @@ static void handle_cap_grant(struct inode *inode,
        BUG_ON(cap->issued & ~cap->implemented);
 
        /* don't let check_caps skip sending a response to MDS for revoke msgs */
-       if (!revoke_wait && le32_to_cpu(grant->op) == CEPH_CAP_OP_REVOKE) {
-               cap->mds_wanted = 0;
-               flags |= CHECK_CAPS_FLUSH_FORCE;
-               if (cap == ci->i_auth_cap)
-                       check_caps = 1; /* check auth cap only */
-               else
-                       check_caps = 2; /* check all caps */
+       if (le32_to_cpu(grant->op) == CEPH_CAP_OP_REVOKE) {
+               if (revoke_wait) {
+                       /*
+                        * We can't ack the revoke yet: the response is deferred
+                        * until the writeback or cache invalidation queued above
+                        * completes.  Set the CEPH_I_FLUSH_FORCE flag to remember
+                        * that a forced cap message is owed so that deferred
+                        * completion (ceph_put_wrbuffer_cap_refs() or the
+                        * invalidate worker, both of which call ceph_check_caps())
+                        * actually sends one, even if by then the revoked caps look
+                        * unused, the inode is retaining caps, or the MDS has
+                        * re-granted them.  Without this, the cap message is never
+                        * sent and the MDS hangs ("isn't responding to
+                        * mclientcaps(revoke)").
+                        */
+                       ci->i_ceph_flags |= CEPH_I_FLUSH_FORCE;
+               } else {
+                       cap->mds_wanted = 0;
+                       flags |= CHECK_CAPS_FLUSH_FORCE;
+                       if (cap == ci->i_auth_cap)
+                               check_caps = 1; /* check auth cap only */
+                       else
+                               check_caps = 2; /* check all caps */
+               }
        }
 
        if (extra_info->inline_version > 0 &&
fs/ceph/super.h +5 -1
index 1d6aab060780..878440a2eb3b 100644
@@ -687,6 +687,10 @@ static inline struct inode *ceph_find_inode(struct super_block *sb,
 #define CEPH_I_ASYNC_CREATE_BIT                (12) /* async create in flight for this */
 #define CEPH_I_SHUTDOWN_BIT            (13) /* inode is no longer usable */
 #define CEPH_I_ASYNC_CHECK_CAPS_BIT    (14) /* check caps after async creating finishes */
+#define CEPH_I_FLUSH_FORCE_BIT         (15) /* a revoke's response was deferred;
+                                             * force a cap message to the MDS once
+                                             * the deferred work completes
+                                             */
 
 #define CEPH_I_DIR_ORDERED             (1 << CEPH_I_DIR_ORDERED_BIT)
 #define CEPH_I_FLUSH                   (1 << CEPH_I_FLUSH_BIT)
@@ -699,6 +703,7 @@ static inline struct inode *ceph_find_inode(struct super_block *sb,
 #define CEPH_I_ODIRECT                 (1 << CEPH_I_ODIRECT_BIT)
 #define CEPH_I_ASYNC_CREATE            (1 << CEPH_I_ASYNC_CREATE_BIT)
 #define CEPH_I_SHUTDOWN                        (1 << CEPH_I_SHUTDOWN_BIT)
+#define CEPH_I_FLUSH_FORCE             (1 << CEPH_I_FLUSH_FORCE_BIT)
 
 /*
  * Masks of ceph inode work.
--

2.47.3

Signature · 2 lines
Cheers,
Venky