ceph: correctly release memory from capsnap
PATCH v3 1/1 ceph: correctly release memory from capsnap
From: Xiubo Li <[email protected]>
When force umounting, it will try to remove all the session caps. If there has any capsnap is in the flushing list, the remove session caps callback will try to release the capsnap->flush_cap memory to "ceph_cap_flush_cachep" slab cache, while which is allocated from kmalloc-256 slab cache.
At the same time switch to list_del_init() because just in case the force umount has removed it from the lists and the handle_cap_flushsnap_ack() comes then the seconds list_del_init() won't crash the kernel.
URL: https://tracker.ceph.com/issues/52283
---
V3:
- rebase to the upstream
fs/ceph/caps.c | 18 ++++++++++++++---- fs/ceph/mds_client.c | 7 ++++--- 2 files changed, 18 insertions(+), 7 deletions(-)
fs/ceph/caps.c +14 -4
@@ -1712,7 +1712,16 @@ int __ceph_mark_dirty_caps(struct ceph_inode_info *ci, int mask,struct ceph_cap_flush *ceph_alloc_cap_flush (void ) { -return kmem_cache_alloc (ceph_cap_flush_cachep,GFP_KERNEL ); +struct ceph_cap_flush *cf; + + cf =kmem_cache_alloc (ceph_cap_flush_cachep,GFP_KERNEL ); +/* +* caps == 0 always means for the capsnap +* caps > 0 means dirty caps being flushed +* caps == -1 means preallocated, not used yet +*/ + cf->caps =-1 ; +return cf; }void ceph_free_cap_flush (struct ceph_cap_flush *cf)
@@ -1747,7 +1756,7 @@ static bool __detach_cap_flush_from_mdsc(struct ceph_mds_client *mdsc, prev->wake = true; wake = false; } -list_del (&cf->g_list); +list_del_init (&cf->g_list);return wake; }
@@ -1762,7 +1771,7 @@ static bool __detach_cap_flush_from_ci(struct ceph_inode_info *ci, prev->wake = true; wake = false; } -list_del (&cf->i_list); +list_del_init (&cf->i_list);return wake; }
@@ -3642,7 +3651,8 @@ static void handle_cap_flush_ack(struct inode *inode, u64 flush_tid, cf =list_first_entry (&to_remove, struct ceph_cap_flush, i_list);list_del (&cf->i_list); -ceph_free_cap_flush (cf); +if (cf->caps) +ceph_free_cap_flush (cf); }if (wake_ci)
fs/ceph/mds_client.c +4 -3
@@ -1636,7 +1636,7 @@ static int remove_session_caps_cb(struct inode *inode, struct ceph_cap *cap,spin_lock (&mdsc->cap_dirty_lock);list_for_each_entry (cf, &to_remove, i_list) -list_del (&cf->g_list); +list_del_init (&cf->g_list);if (!list_empty (&ci->i_dirty_item)) { pr_warn_ratelimited(
@@ -1688,8 +1688,9 @@ static int remove_session_caps_cb(struct inode *inode, struct ceph_cap *cap,struct ceph_cap_flush *cf; cf =list_first_entry (&to_remove, struct ceph_cap_flush, i_list); -list_del (&cf->i_list); -ceph_free_cap_flush (cf); +list_del_init (&cf->i_list); +if (cf->caps) +ceph_free_cap_flush (cf); }wake_up_all (&ci->i_cap_wq);
Signature · 1 line
2.27.0
Ilya Dryomov
maintainerQuoted 33 lines · On Wed, Aug 18, 2021 at 3:25 AM <[email protected]> wrote:
From: Xiubo Li <[email protected]>
When force umounting, it will try to remove all the session caps. If there has any capsnap is in the flushing list, the remove session caps callback will try to release the
capsnap->flush_capmemory to "ceph_cap_flush_cachep" slab cache, while which is allocated from kmalloc-256 slab cache.At the same time switch to
list_del_init()because just in case the force umount has removed it from the lists and thehandle_cap_flushsnap_ack()comes then the secondslist_del_init()won't crash the kernel.URL: https://tracker.ceph.com/issues/52283
Signed-off-by Xiubo Li <[email protected]>---
V3:
- rebase to the upstreamfs/ceph/caps.c | 18 ++++++++++++++---- fs/ceph/mds_client.c | 7 ++++--- 2 files changed, 18 insertions(+), 7 deletions(-)fs/ceph/caps.c +0
-0@@ -1712,7 +1712,16 @@ int __ceph_mark_dirty_caps(struct ceph_inode_info *ci, int mask,struct ceph_cap_flush *ceph_alloc_cap_flush (void )
fs/ceph/caps.c int __ceph_mark_dirty_caps(struct ceph_inode_info *ci, int mask, · line 1722
Quoted 10 lines
fs/ceph/caps.c +8
-1{ -return kmem_cache_alloc (ceph_cap_flush_cachep,GFP_KERNEL ); +struct ceph_cap_flush *cf; + + cf =kmem_cache_alloc (ceph_cap_flush_cachep,GFP_KERNEL ); +/* +* caps == 0 always means for the capsnap +* caps > 0 means dirty caps being flushed +* caps == -1 means preallocated, not used yet +*/
Hi Xiubo,
This comment should be in super.h, on struct ceph_cap_flush definition.
But more importantly, are you sure that overloading cf->caps this way is safe? For example, __kick_flushing_caps() tests for cf->caps != 0 and cf->caps == -1 would be interpreted as a cue to call __prep_cap().
Thanks,
Ilya
Quoted 61 lines
fs/ceph/caps.c +6
-3+ cf->caps =-1 ; +return cf; }void ceph_free_cap_flush (struct ceph_cap_flush *cf)@@ -1747,7 +1756,7 @@ static bool __detach_cap_flush_from_mdsc(struct ceph_mds_client *mdsc, prev->wake = true; wake = false; } -list_del (&cf->g_list); +list_del_init (&cf->g_list);return wake; }@@ -1762,7 +1771,7 @@ static bool __detach_cap_flush_from_ci(struct ceph_inode_info *ci, prev->wake = true; wake = false; } -list_del (&cf->i_list); +list_del_init (&cf->i_list);return wake; }@@ -3642,7 +3651,8 @@ static void handle_cap_flush_ack(struct inode *inode, u64 flush_tid, cf =list_first_entry (&to_remove, struct ceph_cap_flush, i_list);list_del (&cf->i_list); -ceph_free_cap_flush (cf); +if (cf->caps) +ceph_free_cap_flush (cf); }if (wake_ci)fs/ceph/mds_client.c +4
-4@@ -1636,7 +1636,7 @@ static int remove_session_caps_cb(struct inode *inode, struct ceph_cap *cap,spin_lock (&mdsc->cap_dirty_lock);list_for_each_entry (cf, &to_remove, i_list) -list_del (&cf->g_list); +list_del_init (&cf->g_list);if (!list_empty (&ci->i_dirty_item)) { pr_warn_ratelimited(@@ -1688,8 +1688,9 @@ static int remove_session_caps_cb(struct inode *inode, struct ceph_cap *cap,struct ceph_cap_flush *cf; cf =list_first_entry (&to_remove, struct ceph_cap_flush, i_list); -list_del (&cf->i_list); -ceph_free_cap_flush (cf); +list_del_init (&cf->i_list); +if (cf->caps) +ceph_free_cap_flush (cf); }wake_up_all (&ci->i_cap_wq); --2.27.0
Quoted 45 lines · Ilya Dryomov
Quoted 32 lines · On Wed, Aug 18, 2021 at 3:25 AM <[email protected]> wrote:
From: Xiubo Li <[email protected]>
When force umounting, it will try to remove all the session caps. If there has any capsnap is in the flushing list, the remove session caps callback will try to release the
capsnap->flush_capmemory to "ceph_cap_flush_cachep" slab cache, while which is allocated from kmalloc-256 slab cache.At the same time switch to
list_del_init()because just in case the force umount has removed it from the lists and thehandle_cap_flushsnap_ack()comes then the secondslist_del_init()won't crash the kernel.URL: https://tracker.ceph.com/issues/52283
Signed-off-by Xiubo Li <[email protected]>---
V3:
- rebase to the upstreamfs/ceph/caps.c | 18 ++++++++++++++---- fs/ceph/mds_client.c | 7 ++++--- 2 files changed, 18 insertions(+), 7 deletions(-)fs/ceph/caps.c +0
-0@@ -1712,7 +1712,16 @@ int __ceph_mark_dirty_caps(struct ceph_inode_info *ci, int mask,struct ceph_cap_flush *ceph_alloc_cap_flush (void )fs/ceph/caps.c int __ceph_mark_dirty_caps(struct ceph_inode_info *ci, int mask, · line 1722
Quoted 10 lines
fs/ceph/caps.c +8
-1{ -return kmem_cache_alloc (ceph_cap_flush_cachep,GFP_KERNEL ); +struct ceph_cap_flush *cf; + + cf =kmem_cache_alloc (ceph_cap_flush_cachep,GFP_KERNEL ); +/* +* caps == 0 always means for the capsnap +* caps > 0 means dirty caps being flushed +* caps == -1 means preallocated, not used yet +*/ Hi Xiubo,
Quoted 6 lines
This comment should be in
super.h, on structceph_cap_flushdefinition.But more importantly, are you sure that overloading
cf->capsthis way is safe? For example,__kick_flushing_caps()tests forcf->caps!= 0 andcf->caps== -1 would be interpreted as a cue to call__prep_cap().
Hi Ilya,
Yeah, I think it's safe, because once the cf is added into the ci->i_cap_flush_list in __mark_caps_flushing(), it will be guaranteed that the cf->caps will be set some dirty caps, which must be > 0 or it will trigger BUG_ON().
Here in this patch in remove_session_caps_cb() below, the to_remove list will not only pick cf from ci->i_cap_flush_list but also from the ci->i_prealloc_cap_flush, which hasn't been initialized and added to the ci->i_cap_flush_list yet.
Thanks
BRs
Quoted 66 lines
Thanks,
Ilya
Quoted 61 lines
fs/ceph/caps.c +6
-3+ cf->caps =-1 ; +return cf; }void ceph_free_cap_flush (struct ceph_cap_flush *cf)@@ -1747,7 +1756,7 @@ static bool __detach_cap_flush_from_mdsc(struct ceph_mds_client *mdsc, prev->wake = true; wake = false; } -list_del (&cf->g_list); +list_del_init (&cf->g_list);return wake; }@@ -1762,7 +1771,7 @@ static bool __detach_cap_flush_from_ci(struct ceph_inode_info *ci, prev->wake = true; wake = false; } -list_del (&cf->i_list); +list_del_init (&cf->i_list);return wake; }@@ -3642,7 +3651,8 @@ static void handle_cap_flush_ack(struct inode *inode, u64 flush_tid, cf =list_first_entry (&to_remove, struct ceph_cap_flush, i_list);list_del (&cf->i_list); -ceph_free_cap_flush (cf); +if (cf->caps) +ceph_free_cap_flush (cf); }if (wake_ci)fs/ceph/mds_client.c +4
-4@@ -1636,7 +1636,7 @@ static int remove_session_caps_cb(struct inode *inode, struct ceph_cap *cap,spin_lock (&mdsc->cap_dirty_lock);list_for_each_entry (cf, &to_remove, i_list) -list_del (&cf->g_list); +list_del_init (&cf->g_list);if (!list_empty (&ci->i_dirty_item)) { pr_warn_ratelimited(@@ -1688,8 +1688,9 @@ static int remove_session_caps_cb(struct inode *inode, struct ceph_cap *cap,struct ceph_cap_flush *cf; cf =list_first_entry (&to_remove, struct ceph_cap_flush, i_list); -list_del (&cf->i_list); -ceph_free_cap_flush (cf); +list_del_init (&cf->i_list); +if (cf->caps) +ceph_free_cap_flush (cf); }wake_up_all (&ci->i_cap_wq); --2.27.0
Quoted 50 lines · Ilya Dryomov
Quoted 33 lines · On Wed, Aug 18, 2021 at 3:25 AM <[email protected]> wrote:
From: Xiubo Li <[email protected]>
When force umounting, it will try to remove all the session caps. If there has any capsnap is in the flushing list, the remove session caps callback will try to release the
capsnap->flush_capmemory to "ceph_cap_flush_cachep" slab cache, while which is allocated from kmalloc-256 slab cache.At the same time switch to
list_del_init()because just in case the force umount has removed it from the lists and thehandle_cap_flushsnap_ack()comes then the secondslist_del_init()won't crash the kernel.URL: https://tracker.ceph.com/issues/52283
Signed-off-by Xiubo Li <[email protected]>---
V3:
- rebase to the upstreamfs/ceph/caps.c | 18 ++++++++++++++---- fs/ceph/mds_client.c | 7 ++++--- 2 files changed, 18 insertions(+), 7 deletions(-)fs/ceph/caps.c +0
-0@@ -1712,7 +1712,16 @@ int __ceph_mark_dirty_caps(struct ceph_inode_info *ci, int mask,struct ceph_cap_flush *ceph_alloc_cap_flush (void )fs/ceph/caps.c int __ceph_mark_dirty_caps(struct ceph_inode_info *ci, int mask, · line 1722
Quoted 10 lines
fs/ceph/caps.c +8
-1{ -return kmem_cache_alloc (ceph_cap_flush_cachep,GFP_KERNEL ); +struct ceph_cap_flush *cf; + + cf =kmem_cache_alloc (ceph_cap_flush_cachep,GFP_KERNEL ); +/* +* caps == 0 always means for the capsnap +* caps > 0 means dirty caps being flushed +* caps == -1 means preallocated, not used yet +*/ Hi Xiubo,
This comment should be in
super.h, on structceph_cap_flushdefinition.
Quoted 8 lines
But more importantly, are you sure that overloading
cf->capsthis way is safe? For example,__kick_flushing_caps()tests forcf->caps!= 0 andcf->caps== -1 would be interpreted as a cue to call__prep_cap().Thanks,
Ilya
The cf->caps field should get set to a sane value when it goes onto the i_cap_flush_list, and I don't see how we'd get into testing against the ->caps field before that point. That said, this mechanism does seem a bit fragile and subject to later breakage, and the caps code is anything but clear and easy to follow.
pahole says that there is a 3 byte hole just after the "wake" field in ceph_cap_flush on x86_64, and that's probably true on other arches as well. Rather than overloading the caps field with this info, you could add a new bool there to indicate whether it's embedded or not.
Quoted 61 lines
Quoted 61 lines
fs/ceph/caps.c +6
-3+ cf->caps =-1 ; +return cf; }void ceph_free_cap_flush (struct ceph_cap_flush *cf)@@ -1747,7 +1756,7 @@ static bool __detach_cap_flush_from_mdsc(struct ceph_mds_client *mdsc, prev->wake = true; wake = false; } -list_del (&cf->g_list); +list_del_init (&cf->g_list);return wake; }@@ -1762,7 +1771,7 @@ static bool __detach_cap_flush_from_ci(struct ceph_inode_info *ci, prev->wake = true; wake = false; } -list_del (&cf->i_list); +list_del_init (&cf->i_list);return wake; }@@ -3642,7 +3651,8 @@ static void handle_cap_flush_ack(struct inode *inode, u64 flush_tid, cf =list_first_entry (&to_remove, struct ceph_cap_flush, i_list);list_del (&cf->i_list); -ceph_free_cap_flush (cf); +if (cf->caps) +ceph_free_cap_flush (cf); }if (wake_ci)fs/ceph/mds_client.c +4
-4@@ -1636,7 +1636,7 @@ static int remove_session_caps_cb(struct inode *inode, struct ceph_cap *cap,spin_lock (&mdsc->cap_dirty_lock);list_for_each_entry (cf, &to_remove, i_list) -list_del (&cf->g_list); +list_del_init (&cf->g_list);if (!list_empty (&ci->i_dirty_item)) { pr_warn_ratelimited(@@ -1688,8 +1688,9 @@ static int remove_session_caps_cb(struct inode *inode, struct ceph_cap *cap,struct ceph_cap_flush *cf; cf =list_first_entry (&to_remove, struct ceph_cap_flush, i_list); -list_del (&cf->i_list); -ceph_free_cap_flush (cf); +list_del_init (&cf->i_list); +if (cf->caps) +ceph_free_cap_flush (cf); }wake_up_all (&ci->i_cap_wq); --2.27.0
Signature · 1 line
Jeff Layton <[email protected]>
Quoted 63 lines · Jeff Layton
Quoted 48 lines · Ilya Dryomov
Quoted 32 lines · On Wed, Aug 18, 2021 at 3:25 AM <[email protected]> wrote:
From: Xiubo Li <[email protected]>
When force umounting, it will try to remove all the session caps. If there has any capsnap is in the flushing list, the remove session caps callback will try to release the
capsnap->flush_capmemory to "ceph_cap_flush_cachep" slab cache, while which is allocated from kmalloc-256 slab cache.At the same time switch to
list_del_init()because just in case the force umount has removed it from the lists and thehandle_cap_flushsnap_ack()comes then the secondslist_del_init()won't crash the kernel.URL: https://tracker.ceph.com/issues/52283
Signed-off-by Xiubo Li <[email protected]>---
V3:
- rebase to the upstreamfs/ceph/caps.c | 18 ++++++++++++++---- fs/ceph/mds_client.c | 7 ++++--- 2 files changed, 18 insertions(+), 7 deletions(-)fs/ceph/caps.c +0
-0@@ -1712,7 +1712,16 @@ int __ceph_mark_dirty_caps(struct ceph_inode_info *ci, int mask,struct ceph_cap_flush *ceph_alloc_cap_flush (void )fs/ceph/caps.c int __ceph_mark_dirty_caps(struct ceph_inode_info *ci, int mask, · line 1722
Quoted 10 lines
fs/ceph/caps.c +8
-1{ -return kmem_cache_alloc (ceph_cap_flush_cachep,GFP_KERNEL ); +struct ceph_cap_flush *cf; + + cf =kmem_cache_alloc (ceph_cap_flush_cachep,GFP_KERNEL ); +/* +* caps == 0 always means for the capsnap +* caps > 0 means dirty caps being flushed +* caps == -1 means preallocated, not used yet +*/ Hi Xiubo,
This comment should be in
super.h, on structceph_cap_flushdefinition.Quoted 8 lines
But more importantly, are you sure that overloading
cf->capsthis way is safe? For example,__kick_flushing_caps()tests forcf->caps!= 0 andcf->caps== -1 would be interpreted as a cue to call__prep_cap().Thanks,
Ilya
The
cf->capsfield should get set to a sane value when it goes onto thei_cap_flush_list, and I don't see how we'd get into testing against the ->caps field before that point. That said, this mechanism does seem a bit fragile and subject to later breakage, and the caps code is anything but clear and easy to follow.
Quoted 4 lines
pahole says that there is a 3 byte hole just after the "wake" field in
ceph_cap_flushonx86_64, and that's probably true on other arches as well. Rather than overloading the caps field with this info, you could add a new bool there to indicate whether it's embedded or not.
Okay, this also sounds good to me.
I will do that and sent out the V4 later.
Thanks
Quoted 62 lines
Quoted 62 lines
Quoted 62 lines
fs/ceph/caps.c +6
-3+ cf->caps =-1 ; +return cf; }void ceph_free_cap_flush (struct ceph_cap_flush *cf)@@ -1747,7 +1756,7 @@ static bool __detach_cap_flush_from_mdsc(struct ceph_mds_client *mdsc, prev->wake = true; wake = false; } -list_del (&cf->g_list); +list_del_init (&cf->g_list);return wake; }@@ -1762,7 +1771,7 @@ static bool __detach_cap_flush_from_ci(struct ceph_inode_info *ci, prev->wake = true; wake = false; } -list_del (&cf->i_list); +list_del_init (&cf->i_list);return wake; }@@ -3642,7 +3651,8 @@ static void handle_cap_flush_ack(struct inode *inode, u64 flush_tid, cf =list_first_entry (&to_remove, struct ceph_cap_flush, i_list);list_del (&cf->i_list); -ceph_free_cap_flush (cf); +if (cf->caps) +ceph_free_cap_flush (cf); }if (wake_ci)fs/ceph/mds_client.c +4
-4@@ -1636,7 +1636,7 @@ static int remove_session_caps_cb(struct inode *inode, struct ceph_cap *cap,spin_lock (&mdsc->cap_dirty_lock);list_for_each_entry (cf, &to_remove, i_list) -list_del (&cf->g_list); +list_del_init (&cf->g_list);if (!list_empty (&ci->i_dirty_item)) { pr_warn_ratelimited(@@ -1688,8 +1688,9 @@ static int remove_session_caps_cb(struct inode *inode, struct ceph_cap *cap,struct ceph_cap_flush *cf; cf =list_first_entry (&to_remove, struct ceph_cap_flush, i_list); -list_del (&cf->i_list); -ceph_free_cap_flush (cf); +list_del_init (&cf->i_list); +if (cf->caps) +ceph_free_cap_flush (cf); }wake_up_all (&ci->i_cap_wq); --2.27.0