Performance optimization for no fsnotify marks
Subsystems (2)
Amir Goldstein
reviewerv2 cover Performance optimization for no fsnotify marks
Hi Jan,
Following v2 addresses review comments from v1 [1]
[1] https://lore.kernel.org/linux-fsdevel/[email protected]/
Changes since v1:
- Rebase on 5.14-rc5 + pidfd patches
- Added RVB
- Helper to get connector's sb (Matthew)
- Fix deadlock bug on umount (Jan)
Amir Goldstein (4):
fsnotify: replace igrab() with ihold() on attach connector
fsnotify: count s_fsnotify_inode_refs for attached connectors
fsnotify: count all objects with attached connectors
fsnotify: optimize the case of no marks of any type
fs/notify/fsnotify.c | 6 ++--- fs/notify/fsnotify.h | 15 ++++++++++++ fs/notify/mark.c | 52 ++++++++++++++++++++++++++++++---------- include/linux/fs.h | 4 ++-- include/linux/fsnotify.h | 9 +++++++ 5 files changed, 69 insertions(+), 17 deletions(-)
Signature · 1 line
2.32.0
Amir Goldstein
reviewerPATCH v2 1/4 fsnotify: replace igrab() with ihold() on attach connector
We must have a reference on inode, so ihold is cheaper.
--- fs/notify/mark.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)
fs/notify/mark.c +5 -2
@@ -493,8 +493,11 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp, conn->fsid.val[0 ] = conn->fsid.val[1 ] =0 ; conn->flags =0 ; } -if (conn->type ==FSNOTIFY_OBJ_TYPE_INODE ) - inode =igrab (fsnotify_conn_inode (conn)); +if (conn->type ==FSNOTIFY_OBJ_TYPE_INODE ) { + inode =fsnotify_conn_inode (conn); +ihold (inode); + } +/* * cmpxchg() provides the barrier so that readers of *connp can see * only initialized structure
Signature · 1 line
2.32.0
Amir Goldstein
reviewerPATCH v2 2/4 fsnotify: count s_fsnotify_inode_refs for attached connectors
In reply to Amir GoldsteinInstead of incrementing s_fsnotify_inode_refs when detaching connector from inode, increment it earlier when attaching connector to inode. Next patch is going to use s_fsnotify_inode_refs to count all objects with attached connectors.
--- fs/notify/mark.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-)
fs/notify/mark.c +18 -11
@@ -169,6 +169,21 @@ static void fsnotify_connector_destroy_workfn(struct work_struct *work) } } +static void fsnotify_get_inode_ref (struct inode *inode) +{ +ihold (inode); +atomic_long_inc (&inode->i_sb->s_fsnotify_inode_refs); +} + +static void fsnotify_put_inode_ref (struct inode *inode) +{ +struct super_block *sb = inode->i_sb; + +iput (inode); +if (atomic_long_dec_and_test (&sb->s_fsnotify_inode_refs)) +wake_up_var (&sb->s_fsnotify_inode_refs); +} +static void *fsnotify_detach_connector_from_object (struct fsnotify_mark_connector *conn,unsigned int *type)
@@ -182,7 +197,6 @@ static void *fsnotify_detach_connector_from_object(if (conn->type ==FSNOTIFY_OBJ_TYPE_INODE ) { inode =fsnotify_conn_inode (conn); inode->i_fsnotify_mask =0 ; -atomic_long_inc (&inode->i_sb->s_fsnotify_inode_refs); }else if (conn->type ==FSNOTIFY_OBJ_TYPE_VFSMOUNT ) {fsnotify_conn_mount (conn)->mnt_fsnotify_mask =0 ; }else if (conn->type ==FSNOTIFY_OBJ_TYPE_SB ) {
@@ -209,19 +223,12 @@ static void fsnotify_final_mark_destroy(struct fsnotify_mark *mark)/* Drop object reference originally held by a connector */ static void fsnotify_drop_object (unsigned int type,void *objp) { -struct inode *inode; -struct super_block *sb; -if (!objp)return ;/* Currently only inode references are passed to be dropped */ if (WARN_ON_ONCE (type !=FSNOTIFY_OBJ_TYPE_INODE ))return ; - inode = objp; - sb = inode->i_sb; -iput (inode); -if (atomic_long_dec_and_test (&sb->s_fsnotify_inode_refs)) -wake_up_var (&sb->s_fsnotify_inode_refs); +fsnotify_put_inode_ref (objp); }void fsnotify_put_mark (struct fsnotify_mark *mark)
@@ -495,7 +502,7 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp, }if (conn->type ==FSNOTIFY_OBJ_TYPE_INODE ) { inode =fsnotify_conn_inode (conn); -ihold (inode); +fsnotify_get_inode_ref (inode); }/*
@@ -505,7 +512,7 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp,if (cmpxchg (connp,NULL , conn)) {/* Someone else created list structure for us */ if (inode) -iput (inode); +fsnotify_put_inode_ref (inode);kmem_cache_free (fsnotify_mark_connector_cachep, conn); }
Signature · 1 line
2.32.0
Amir Goldstein
reviewerPATCH v2 3/4 fsnotify: count all objects with attached connectors
In reply to Amir GoldsteinRename s_fsnotify_inode_refs to s_fsnotify_connectors and count all objects with attached connectors, not only inodes with attached connectors.
This will be used to optimize fsnotify() calls on sb without any type of marks.
--- fs/notify/fsnotify.c | 6 +++--- fs/notify/fsnotify.h | 15 +++++++++++++++ fs/notify/mark.c | 24 +++++++++++++++++++++--- include/linux/fs.h | 4 ++-- 4 files changed, 41 insertions(+), 8 deletions(-)
fs/notify/fsnotify.c +3 -3
@@ -87,15 +87,15 @@ static void fsnotify_unmount_inodes(struct super_block *sb)if (iput_inode)iput (iput_inode); -/* Wait for outstanding inode references from connectors */ -wait_var_event (&sb->s_fsnotify_inode_refs, - !atomic_long_read (&sb->s_fsnotify_inode_refs)); }void fsnotify_sb_delete (struct super_block *sb) {fsnotify_unmount_inodes (sb);fsnotify_clear_marks_by_sb (sb); +/* Wait for outstanding object references from connectors */ +wait_var_event (&sb->s_fsnotify_connectors, + !atomic_long_read (&sb->s_fsnotify_connectors)); }/*
fs/notify/fsnotify.h +15 -0
@@ -27,6 +27,21 @@ static inline struct super_block *fsnotify_conn_sb( return container_of(conn->obj, struct super_block, s_fsnotify_marks); } +static inline struct super_block *fsnotify_connector_sb ( +struct fsnotify_mark_connector *conn) +{ +switch (conn->type) { +case FSNOTIFY_OBJ_TYPE_INODE : +return fsnotify_conn_inode (conn)->i_sb; +case FSNOTIFY_OBJ_TYPE_VFSMOUNT : +return fsnotify_conn_mount (conn)->mnt.mnt_sb; +case FSNOTIFY_OBJ_TYPE_SB : +return fsnotify_conn_sb (conn); +default : +return NULL ; + } +} +/* destroy all events sitting in this groups notification queue */ extern void fsnotify_flush_notify (struct fsnotify_group *group);
fs/notify/mark.c +21 -3
@@ -172,7 +172,7 @@ static void fsnotify_connector_destroy_workfn(struct work_struct *work)static void fsnotify_get_inode_ref (struct inode *inode) {ihold (inode); -atomic_long_inc (&inode->i_sb->s_fsnotify_inode_refs); +atomic_long_inc (&inode->i_sb->s_fsnotify_connectors); }static void fsnotify_put_inode_ref (struct inode *inode)
@@ -180,8 +180,24 @@ static void fsnotify_put_inode_ref(struct inode *inode)struct super_block *sb = inode->i_sb;iput (inode); -if (atomic_long_dec_and_test (&sb->s_fsnotify_inode_refs)) -wake_up_var (&sb->s_fsnotify_inode_refs); +if (atomic_long_dec_and_test (&sb->s_fsnotify_connectors)) +wake_up_var (&sb->s_fsnotify_connectors); +} + +static void fsnotify_get_sb_connectors (struct fsnotify_mark_connector *conn) +{ +struct super_block *sb =fsnotify_connector_sb (conn); + +if (sb) +atomic_long_inc (&sb->s_fsnotify_connectors); +} + +static void fsnotify_put_sb_connectors (struct fsnotify_mark_connector *conn) +{ +struct super_block *sb =fsnotify_connector_sb (conn); + +if (sb &&atomic_long_dec_and_test (&sb->s_fsnotify_connectors)) +wake_up_var (&sb->s_fsnotify_connectors); } static void *fsnotify_detach_connector_from_object(
@@ -203,6 +219,7 @@ static void *fsnotify_detach_connector_from_object(fsnotify_conn_sb (conn)->s_fsnotify_mask =0 ; } +fsnotify_put_sb_connectors (conn);rcu_assign_pointer (*(conn->obj),NULL ); conn->obj =NULL ; conn->type =FSNOTIFY_OBJ_TYPE_DETACHED ;
@@ -504,6 +521,7 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp, inode =fsnotify_conn_inode (conn);fsnotify_get_inode_ref (inode); } +fsnotify_get_sb_connectors (conn);/* * cmpxchg() provides the barrier so that readers of *connp can see
include/linux/fs.h +2 -2
@@ -1507,8 +1507,8 @@ struct super_block {/* Number of inodes with nlink == 0 but still referenced */ atomic_long_t s_remove_count; -/* Pending fsnotify inode refs */ -atomic_long_t s_fsnotify_inode_refs; +/* Number of inode/mount/sb objects that are being watched */ +atomic_long_t s_fsnotify_connectors;/* Being remounted read-only */ int s_readonly_remount;
Signature · 1 line
2.32.0
Re: [PATCH v2 3/4] fsnotify: count all objects with attached connectors
Quoted 8 lines · Amir Goldstein
Rename
s_fsnotify_inode_refstos_fsnotify_connectorsand count all objects with attached connectors, not only inodes with attached connectors.This will be used to optimize
fsnotify()calls on sb without any type of marks.Signed-off-by Amir Goldstein <[email protected]>
LGTM.
Quoted 130 lines
--- fs/notify/fsnotify.c | 6 +++--- fs/notify/fsnotify.h | 15 +++++++++++++++ fs/notify/mark.c | 24 +++++++++++++++++++++--- include/linux/fs.h | 4 ++-- 4 files changed, 41 insertions(+), 8 deletions(-)fs/notify/fsnotify.c +3
-3@@ -87,15 +87,15 @@ static void fsnotify_unmount_inodes(struct super_block *sb)if (iput_inode)iput (iput_inode); -/* Wait for outstanding inode references from connectors */ -wait_var_event (&sb->s_fsnotify_inode_refs, - !atomic_long_read (&sb->s_fsnotify_inode_refs)); }void fsnotify_sb_delete (struct super_block *sb) {fsnotify_unmount_inodes (sb);fsnotify_clear_marks_by_sb (sb); +/* Wait for outstanding object references from connectors */ +wait_var_event (&sb->s_fsnotify_connectors, + !atomic_long_read (&sb->s_fsnotify_connectors)); }/* fs/notify/fsnotify.h +15
-0@@ -27,6 +27,21 @@ static inline struct super_block *fsnotify_conn_sb( return container_of(conn->obj, struct super_block, s_fsnotify_marks); } +static inline struct super_block *fsnotify_connector_sb ( +struct fsnotify_mark_connector *conn) +{ +switch (conn->type) { +case FSNOTIFY_OBJ_TYPE_INODE : +return fsnotify_conn_inode (conn)->i_sb; +case FSNOTIFY_OBJ_TYPE_VFSMOUNT : +return fsnotify_conn_mount (conn)->mnt.mnt_sb; +case FSNOTIFY_OBJ_TYPE_SB : +return fsnotify_conn_sb (conn); +default : +return NULL ; + } +} +/* destroy all events sitting in this groups notification queue */ extern void fsnotify_flush_notify (struct fsnotify_group *group);fs/notify/mark.c +21
-3@@ -172,7 +172,7 @@ static void fsnotify_connector_destroy_workfn(struct work_struct *work)static void fsnotify_get_inode_ref (struct inode *inode) {ihold (inode); -atomic_long_inc (&inode->i_sb->s_fsnotify_inode_refs); +atomic_long_inc (&inode->i_sb->s_fsnotify_connectors); }static void fsnotify_put_inode_ref (struct inode *inode)@@ -180,8 +180,24 @@ static void fsnotify_put_inode_ref(struct inode *inode)struct super_block *sb = inode->i_sb;iput (inode); -if (atomic_long_dec_and_test (&sb->s_fsnotify_inode_refs)) -wake_up_var (&sb->s_fsnotify_inode_refs); +if (atomic_long_dec_and_test (&sb->s_fsnotify_connectors)) +wake_up_var (&sb->s_fsnotify_connectors); +} + +static void fsnotify_get_sb_connectors (struct fsnotify_mark_connector *conn) +{ +struct super_block *sb =fsnotify_connector_sb (conn); + +if (sb) +atomic_long_inc (&sb->s_fsnotify_connectors); +} + +static void fsnotify_put_sb_connectors (struct fsnotify_mark_connector *conn) +{ +struct super_block *sb =fsnotify_connector_sb (conn); + +if (sb &&atomic_long_dec_and_test (&sb->s_fsnotify_connectors)) +wake_up_var (&sb->s_fsnotify_connectors); } static void *fsnotify_detach_connector_from_object(@@ -203,6 +219,7 @@ static void *fsnotify_detach_connector_from_object(fsnotify_conn_sb (conn)->s_fsnotify_mask =0 ; } +fsnotify_put_sb_connectors (conn);rcu_assign_pointer (*(conn->obj),NULL ); conn->obj =NULL ; conn->type =FSNOTIFY_OBJ_TYPE_DETACHED ;@@ -504,6 +521,7 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp, inode =fsnotify_conn_inode (conn);fsnotify_get_inode_ref (inode); } +fsnotify_get_sb_connectors (conn);/* * cmpxchg() provides the barrier so that readers of *connp can see include/linux/fs.h +2
-2@@ -1507,8 +1507,8 @@ struct super_block {/* Number of inodes with nlink == 0 but still referenced */ atomic_long_t s_remove_count; -/* Pending fsnotify inode refs */ -atomic_long_t s_fsnotify_inode_refs; +/* Number of inode/mount/sb objects that are being watched */ +atomic_long_t s_fsnotify_connectors;/* Being remounted read-only */ int s_readonly_remount;Signature · 1 line
2.32.0
/M
Jan Kara
maintainerRe: [PATCH v2 3/4] fsnotify: count all objects with attached connectors
In reply to Amir GoldsteinQuoted 8 lines · Amir Goldstein
Rename
s_fsnotify_inode_refstos_fsnotify_connectorsand count all objects with attached connectors, not only inodes with attached connectors.This will be used to optimize
fsnotify()calls on sb without any type of marks.Signed-off-by Amir Goldstein <[email protected]>
... just a minor nit below ...
include/linux/fs.h struct super_block { · line 1511
Quoted 12 lines
include/linux/fs.h +2
-2@@ -1507,8 +1507,8 @@ struct super_block {/* Number of inodes with nlink == 0 but still referenced */ atomic_long_t s_remove_count; -/* Pending fsnotify inode refs */ -atomic_long_t s_fsnotify_inode_refs; +/* Number of inode/mount/sb objects that are being watched */ +atomic_long_t s_fsnotify_connectors;
I've realized inode watches will be double-accounted because we increment s_fsnotify_connectors both when attaching a connector and when grabbing inode reference. It doesn't really matter and avoiding this would require special treatment of inode connectors (we need to decrement s_fsnotify_connectors only after dropping inode reference). So I'll just reflect this in the comment here so that we don't forget.
Honza
Signature · 2 lines
Jan Kara <[email protected]> SUSE Labs, CR
Amir Goldstein
reviewerRe: [PATCH v2 3/4] fsnotify: count all objects with attached connectors
Quoted 21 lines · Jan Kara
Quoted 8 lines · Amir Goldstein
Rename
s_fsnotify_inode_refstos_fsnotify_connectorsand count all objects with attached connectors, not only inodes with attached connectors.This will be used to optimize
fsnotify()calls on sb without any type of marks.Signed-off-by Amir Goldstein <[email protected]>... just a minor nit below ...
Quoted 8 lines
include/linux/fs.h +0
-0@@ -1507,8 +1507,8 @@ struct super_block {/* Number of inodes with nlink == 0 but still referenced */ atomic_long_t s_remove_count;
Quoted 11 lines
include/linux/fs.h struct super_block { · line 1511
Quoted 4 lines
include/linux/fs.h +2
-2-/* Pending fsnotify inode refs */ -atomic_long_t s_fsnotify_inode_refs; +/* Number of inode/mount/sb objects that are being watched */ +atomic_long_t s_fsnotify_connectors;I've realized inode watches will be double-accounted because we increment
s_fsnotify_connectorsboth when attaching a connector and when grabbing inode reference. It doesn't really matter and avoiding this would require special treatment of inode connectors (we need to decrements_fsnotify_connectorsonly after dropping inode reference). So I'll just reflect this in the comment here so that we don't forget.
Great!
I actually have a patch to make the inode reference optional per backend/mark [1], so keeping double account makes things a lot easier for that code as well.
Thanks,
Amir.
[1] https://github.com/amir73il/linux/commits/fsnotify-volatile
Amir Goldstein
reviewerPATCH v2 4/4 fsnotify: optimize the case of no marks of any type
In reply to Amir GoldsteinAdd a simple check in the inline helpers to avoid calling fsnotify() and __fsnotify_parent() in case there are no marks of any type (inode/sb/mount) for an inode's sb, so there can be no objects of any type interested in the event.
--- include/linux/fsnotify.h | 9 +++++++++ 1 file changed, 9 insertions(+)
include/linux/fsnotify.h +9 -0
@@ -30,6 +30,9 @@ static inline void fsnotify_name(struct inode *dir, __u32 mask,struct inode *child, const struct qstr *name, u32 cookie) { + if (atomic_long_read(&dir->i_sb->s_fsnotify_connectors) == 0) + return; +fsnotify (mask ,child ,FSNOTIFY_EVENT_INODE ,dir ,name ,NULL ,cookie ); }
@@ -41,6 +44,9 @@ static inline void fsnotify_dirent(struct inode *dir, struct dentry *dentry,static inline void fsnotify_inode (struct inode *inode,__u32 mask) { +if (atomic_long_read (&inode->i_sb->s_fsnotify_connectors) ==0 ) +return ; +if (S_ISDIR (inode->i_mode)) mask |=FS_ISDIR ;
@@ -53,6 +59,9 @@ static inline int fsnotify_parent(struct dentry *dentry, __u32 mask, {struct inode *inode =d_inode (dentry); +if (atomic_long_read (&inode->i_sb->s_fsnotify_connectors) ==0 ) +return 0 ; +if (S_ISDIR (inode->i_mode)) { mask |=FS_ISDIR ;
Signature · 1 line
2.32.0
Jan Kara
maintainerQuoted 5 lines · Amir Goldstein
Hi Jan,
Following v2 addresses review comments from v1 [1]
[1] https://lore.kernel.org/linux-fsdevel/[email protected]/
Thanks for the patches and for the review to Matthew. I've picked up the patches to my tree.
Honza
Quoted 23 lines
Changes since v1:
- Rebase on 5.14-rc5 + pidfd patches
- Added RVB
- Helper to get connector's sb (Matthew)
- Fix deadlock bug on umount (Jan)Amir Goldstein (4):
fsnotify: replaceigrab()withihold()on attach connector
fsnotify: counts_fsnotify_inode_refsfor attached connectors
fsnotify: count all objects with attached connectors
fsnotify: optimize the case of no marks of any typefs/notify/fsnotify.c | 6 ++--- fs/notify/fsnotify.h | 15 ++++++++++++ fs/notify/mark.c | 52 ++++++++++++++++++++++++++++++---------- include/linux/fs.h | 4 ++-- include/linux/fsnotify.h | 9 +++++++ 5 files changed, 69 insertions(+), 17 deletions(-)Signature · 1 line
2.32.0
Signature · 2 lines
Jan Kara <[email protected]> SUSE Labs, CR