Skip to contentSkip to conversation

Performance optimization for no fsnotify marks

PATCH In review Amir Goldstein 3 participants 9 messages Aug 10, 2021 – Aug 11, 2021

Subsystems (2)
lore
Share & tools
lore

Amir Goldstein

author reviewer
#0 lore raw

v2 cover Performance optimization for no fsnotify marks

Hi Jan,

Following v2 addresses review comments from v1 [1]

[1] https://lore.kernel.org/linux-fsdevel/[email protected]/

Changes since v1:
- Rebase on 5.14-rc5 + pidfd patches
- Added RVB
- Helper to get connector's sb (Matthew)
- Fix deadlock bug on umount (Jan)

Amir Goldstein (4):
fsnotify: replace igrab() with ihold() on attach connector
fsnotify: count s_fsnotify_inode_refs for attached connectors
fsnotify: count all objects with attached connectors
fsnotify: optimize the case of no marks of any type

 fs/notify/fsnotify.c     |  6 ++---
 fs/notify/fsnotify.h     | 15 ++++++++++++
 fs/notify/mark.c         | 52 ++++++++++++++++++++++++++++++----------
 include/linux/fs.h       |  4 ++--
 include/linux/fsnotify.h |  9 +++++++
 5 files changed, 69 insertions(+), 17 deletions(-)
Signature · 1 line
2.32.0
author reviewer
#1 lore raw

PATCH v2 1/4 fsnotify: replace igrab() with ihold() on attach connector

We must have a reference on inode, so ihold is cheaper.

---
 fs/notify/mark.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)
fs/notify/mark.c +5 -2
index d32ab349db74..80459db58f63 100644
@@ -493,8 +493,11 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp,
 		conn->fsid.val[0] = conn->fsid.val[1] = 0;
 		conn->flags = 0;
 	}
-	if (conn->type == FSNOTIFY_OBJ_TYPE_INODE)
-		inode = igrab(fsnotify_conn_inode(conn));
+	if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) {
+		inode = fsnotify_conn_inode(conn);
+		ihold(inode);
+	}
+
 	/*
 	 * cmpxchg() provides the barrier so that readers of *connp can see
 	 * only initialized structure
Signature · 1 line
2.32.0
author reviewer
#2 lore raw

PATCH v2 2/4 fsnotify: count s_fsnotify_inode_refs for attached connectors

In reply to Amir Goldstein

Instead of incrementing s_fsnotify_inode_refs when detaching connector from inode, increment it earlier when attaching connector to inode. Next patch is going to use s_fsnotify_inode_refs to count all objects with attached connectors.

---
 fs/notify/mark.c | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)
fs/notify/mark.c +18 -11
index 80459db58f63..2d8c46e1167d 100644
@@ -169,6 +169,21 @@ static void fsnotify_connector_destroy_workfn(struct work_struct *work)
 	}
 }
 
+static void fsnotify_get_inode_ref(struct inode *inode)
+{
+	ihold(inode);
+	atomic_long_inc(&inode->i_sb->s_fsnotify_inode_refs);
+}
+
+static void fsnotify_put_inode_ref(struct inode *inode)
+{
+	struct super_block *sb = inode->i_sb;
+
+	iput(inode);
+	if (atomic_long_dec_and_test(&sb->s_fsnotify_inode_refs))
+		wake_up_var(&sb->s_fsnotify_inode_refs);
+}
+
 static void *fsnotify_detach_connector_from_object(
 					struct fsnotify_mark_connector *conn,
 					unsigned int *type)
@@ -182,7 +197,6 @@ static void *fsnotify_detach_connector_from_object(
 	if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) {
 		inode = fsnotify_conn_inode(conn);
 		inode->i_fsnotify_mask = 0;
-		atomic_long_inc(&inode->i_sb->s_fsnotify_inode_refs);
 	} else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) {
 		fsnotify_conn_mount(conn)->mnt_fsnotify_mask = 0;
 	} else if (conn->type == FSNOTIFY_OBJ_TYPE_SB) {
@@ -209,19 +223,12 @@ static void fsnotify_final_mark_destroy(struct fsnotify_mark *mark)
 /* Drop object reference originally held by a connector */
 static void fsnotify_drop_object(unsigned int type, void *objp)
 {
-	struct inode *inode;
-	struct super_block *sb;
-
 	if (!objp)
 		return;
 	/* Currently only inode references are passed to be dropped */
 	if (WARN_ON_ONCE(type != FSNOTIFY_OBJ_TYPE_INODE))
 		return;
-	inode = objp;
-	sb = inode->i_sb;
-	iput(inode);
-	if (atomic_long_dec_and_test(&sb->s_fsnotify_inode_refs))
-		wake_up_var(&sb->s_fsnotify_inode_refs);
+	fsnotify_put_inode_ref(objp);
 }
 
 void fsnotify_put_mark(struct fsnotify_mark *mark)
@@ -495,7 +502,7 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp,
 	}
 	if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) {
 		inode = fsnotify_conn_inode(conn);
-		ihold(inode);
+		fsnotify_get_inode_ref(inode);
 	}
 
 	/*
@@ -505,7 +512,7 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp,
 	if (cmpxchg(connp, NULL, conn)) {
 		/* Someone else created list structure for us */
 		if (inode)
-			iput(inode);
+			fsnotify_put_inode_ref(inode);
 		kmem_cache_free(fsnotify_mark_connector_cachep, conn);
 	}
 
Signature · 1 line
2.32.0
author reviewer
#3 lore raw

PATCH v2 3/4 fsnotify: count all objects with attached connectors

In reply to Amir Goldstein

Rename s_fsnotify_inode_refs to s_fsnotify_connectors and count all objects with attached connectors, not only inodes with attached connectors.

This will be used to optimize fsnotify() calls on sb without any type of marks.

---
 fs/notify/fsnotify.c |  6 +++---
 fs/notify/fsnotify.h | 15 +++++++++++++++
 fs/notify/mark.c     | 24 +++++++++++++++++++++---
 include/linux/fs.h   |  4 ++--
 4 files changed, 41 insertions(+), 8 deletions(-)
fs/notify/fsnotify.c +3 -3
index 30d422b8c0fc..963e6ce75b96 100644
@@ -87,15 +87,15 @@ static void fsnotify_unmount_inodes(struct super_block *sb)
 
 	if (iput_inode)
 		iput(iput_inode);
-	/* Wait for outstanding inode references from connectors */
-	wait_var_event(&sb->s_fsnotify_inode_refs,
-		       !atomic_long_read(&sb->s_fsnotify_inode_refs));
 }
 
 void fsnotify_sb_delete(struct super_block *sb)
 {
 	fsnotify_unmount_inodes(sb);
 	fsnotify_clear_marks_by_sb(sb);
+	/* Wait for outstanding object references from connectors */
+	wait_var_event(&sb->s_fsnotify_connectors,
+		       !atomic_long_read(&sb->s_fsnotify_connectors));
 }
 
 /*
fs/notify/fsnotify.h +15 -0
index ff2063ec6b0f..87d8a50ee803 100644
@@ -27,6 +27,21 @@ static inline struct super_block *fsnotify_conn_sb(
 	return container_of(conn->obj, struct super_block, s_fsnotify_marks);
 }
 
+static inline struct super_block *fsnotify_connector_sb(
+				struct fsnotify_mark_connector *conn)
+{
+	switch (conn->type) {
+	case FSNOTIFY_OBJ_TYPE_INODE:
+		return fsnotify_conn_inode(conn)->i_sb;
+	case FSNOTIFY_OBJ_TYPE_VFSMOUNT:
+		return fsnotify_conn_mount(conn)->mnt.mnt_sb;
+	case FSNOTIFY_OBJ_TYPE_SB:
+		return fsnotify_conn_sb(conn);
+	default:
+		return NULL;
+	}
+}
+
 /* destroy all events sitting in this groups notification queue */
 extern void fsnotify_flush_notify(struct fsnotify_group *group);
 
fs/notify/mark.c +21 -3
index 2d8c46e1167d..95006d1d29ab 100644
@@ -172,7 +172,7 @@ static void fsnotify_connector_destroy_workfn(struct work_struct *work)
 static void fsnotify_get_inode_ref(struct inode *inode)
 {
 	ihold(inode);
-	atomic_long_inc(&inode->i_sb->s_fsnotify_inode_refs);
+	atomic_long_inc(&inode->i_sb->s_fsnotify_connectors);
 }
 
 static void fsnotify_put_inode_ref(struct inode *inode)
@@ -180,8 +180,24 @@ static void fsnotify_put_inode_ref(struct inode *inode)
 	struct super_block *sb = inode->i_sb;
 
 	iput(inode);
-	if (atomic_long_dec_and_test(&sb->s_fsnotify_inode_refs))
-		wake_up_var(&sb->s_fsnotify_inode_refs);
+	if (atomic_long_dec_and_test(&sb->s_fsnotify_connectors))
+		wake_up_var(&sb->s_fsnotify_connectors);
+}
+
+static void fsnotify_get_sb_connectors(struct fsnotify_mark_connector *conn)
+{
+	struct super_block *sb = fsnotify_connector_sb(conn);
+
+	if (sb)
+		atomic_long_inc(&sb->s_fsnotify_connectors);
+}
+
+static void fsnotify_put_sb_connectors(struct fsnotify_mark_connector *conn)
+{
+	struct super_block *sb = fsnotify_connector_sb(conn);
+
+	if (sb && atomic_long_dec_and_test(&sb->s_fsnotify_connectors))
+		wake_up_var(&sb->s_fsnotify_connectors);
 }
 
 static void *fsnotify_detach_connector_from_object(
@@ -203,6 +219,7 @@ static void *fsnotify_detach_connector_from_object(
 		fsnotify_conn_sb(conn)->s_fsnotify_mask = 0;
 	}
 
+	fsnotify_put_sb_connectors(conn);
 	rcu_assign_pointer(*(conn->obj), NULL);
 	conn->obj = NULL;
 	conn->type = FSNOTIFY_OBJ_TYPE_DETACHED;
@@ -504,6 +521,7 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp,
 		inode = fsnotify_conn_inode(conn);
 		fsnotify_get_inode_ref(inode);
 	}
+	fsnotify_get_sb_connectors(conn);
 
 	/*
 	 * cmpxchg() provides the barrier so that readers of *connp can see
include/linux/fs.h +2 -2
index 640574294216..d48d2018dfa4 100644
@@ -1507,8 +1507,8 @@ struct super_block {
 	/* Number of inodes with nlink == 0 but still referenced */
 	atomic_long_t s_remove_count;
 
-	/* Pending fsnotify inode refs */
-	atomic_long_t s_fsnotify_inode_refs;
+	/* Number of inode/mount/sb objects that are being watched */
+	atomic_long_t s_fsnotify_connectors;
 
 	/* Being remounted read-only */
 	int s_readonly_remount;
Signature · 1 line
2.32.0
Reviewed-by on v2 3/4
#4 lore raw

Re: [PATCH v2 3/4] fsnotify: count all objects with attached connectors

Quoted 8 lines · Amir Goldstein

Rename s_fsnotify_inode_refs to s_fsnotify_connectors and count all objects with attached connectors, not only inodes with attached connectors.

This will be used to optimize fsnotify() calls on sb without any type of marks.

LGTM.

Quoted 130 lines
---
 fs/notify/fsnotify.c |  6 +++---
 fs/notify/fsnotify.h | 15 +++++++++++++++
 fs/notify/mark.c     | 24 +++++++++++++++++++++---
 include/linux/fs.h   |  4 ++--
 4 files changed, 41 insertions(+), 8 deletions(-)
fs/notify/fsnotify.c +3 -3
index 30d422b8c0fc..963e6ce75b96 100644
@@ -87,15 +87,15 @@ static void fsnotify_unmount_inodes(struct super_block *sb)
 
 	if (iput_inode)
 		iput(iput_inode);
-	/* Wait for outstanding inode references from connectors */
-	wait_var_event(&sb->s_fsnotify_inode_refs,
-		       !atomic_long_read(&sb->s_fsnotify_inode_refs));
 }
 
 void fsnotify_sb_delete(struct super_block *sb)
 {
 	fsnotify_unmount_inodes(sb);
 	fsnotify_clear_marks_by_sb(sb);
+	/* Wait for outstanding object references from connectors */
+	wait_var_event(&sb->s_fsnotify_connectors,
+		       !atomic_long_read(&sb->s_fsnotify_connectors));
 }
 
 /*
fs/notify/fsnotify.h +15 -0
index ff2063ec6b0f..87d8a50ee803 100644
@@ -27,6 +27,21 @@ static inline struct super_block *fsnotify_conn_sb(
 	return container_of(conn->obj, struct super_block, s_fsnotify_marks);
 }
 
+static inline struct super_block *fsnotify_connector_sb(
+				struct fsnotify_mark_connector *conn)
+{
+	switch (conn->type) {
+	case FSNOTIFY_OBJ_TYPE_INODE:
+		return fsnotify_conn_inode(conn)->i_sb;
+	case FSNOTIFY_OBJ_TYPE_VFSMOUNT:
+		return fsnotify_conn_mount(conn)->mnt.mnt_sb;
+	case FSNOTIFY_OBJ_TYPE_SB:
+		return fsnotify_conn_sb(conn);
+	default:
+		return NULL;
+	}
+}
+
 /* destroy all events sitting in this groups notification queue */
 extern void fsnotify_flush_notify(struct fsnotify_group *group);
 
fs/notify/mark.c +21 -3
index 2d8c46e1167d..95006d1d29ab 100644
@@ -172,7 +172,7 @@ static void fsnotify_connector_destroy_workfn(struct work_struct *work)
 static void fsnotify_get_inode_ref(struct inode *inode)
 {
 	ihold(inode);
-	atomic_long_inc(&inode->i_sb->s_fsnotify_inode_refs);
+	atomic_long_inc(&inode->i_sb->s_fsnotify_connectors);
 }
 
 static void fsnotify_put_inode_ref(struct inode *inode)
@@ -180,8 +180,24 @@ static void fsnotify_put_inode_ref(struct inode *inode)
 	struct super_block *sb = inode->i_sb;
 
 	iput(inode);
-	if (atomic_long_dec_and_test(&sb->s_fsnotify_inode_refs))
-		wake_up_var(&sb->s_fsnotify_inode_refs);
+	if (atomic_long_dec_and_test(&sb->s_fsnotify_connectors))
+		wake_up_var(&sb->s_fsnotify_connectors);
+}
+
+static void fsnotify_get_sb_connectors(struct fsnotify_mark_connector *conn)
+{
+	struct super_block *sb = fsnotify_connector_sb(conn);
+
+	if (sb)
+		atomic_long_inc(&sb->s_fsnotify_connectors);
+}
+
+static void fsnotify_put_sb_connectors(struct fsnotify_mark_connector *conn)
+{
+	struct super_block *sb = fsnotify_connector_sb(conn);
+
+	if (sb && atomic_long_dec_and_test(&sb->s_fsnotify_connectors))
+		wake_up_var(&sb->s_fsnotify_connectors);
 }
 
 static void *fsnotify_detach_connector_from_object(
@@ -203,6 +219,7 @@ static void *fsnotify_detach_connector_from_object(
 		fsnotify_conn_sb(conn)->s_fsnotify_mask = 0;
 	}
 
+	fsnotify_put_sb_connectors(conn);
 	rcu_assign_pointer(*(conn->obj), NULL);
 	conn->obj = NULL;
 	conn->type = FSNOTIFY_OBJ_TYPE_DETACHED;
@@ -504,6 +521,7 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp,
 		inode = fsnotify_conn_inode(conn);
 		fsnotify_get_inode_ref(inode);
 	}
+	fsnotify_get_sb_connectors(conn);
 
 	/*
 	 * cmpxchg() provides the barrier so that readers of *connp can see
include/linux/fs.h +2 -2
index 640574294216..d48d2018dfa4 100644
@@ -1507,8 +1507,8 @@ struct super_block {
 	/* Number of inodes with nlink == 0 but still referenced */
 	atomic_long_t s_remove_count;
 
-	/* Pending fsnotify inode refs */
-	atomic_long_t s_fsnotify_inode_refs;
+	/* Number of inode/mount/sb objects that are being watched */
+	atomic_long_t s_fsnotify_connectors;
 
 	/* Being remounted read-only */
 	int s_readonly_remount;
Signature · 1 line
2.32.0

/M

maintainer
#5 lore raw

Re: [PATCH v2 3/4] fsnotify: count all objects with attached connectors

In reply to Amir Goldstein
Quoted 8 lines · Amir Goldstein

Rename s_fsnotify_inode_refs to s_fsnotify_connectors and count all objects with attached connectors, not only inodes with attached connectors.

This will be used to optimize fsnotify() calls on sb without any type of marks.

... just a minor nit below ...

include/linux/fs.h struct super_block { · line 1511

Quoted 12 lines
include/linux/fs.h +2 -2
index 640574294216..d48d2018dfa4 100644
@@ -1507,8 +1507,8 @@ struct super_block {
 	/* Number of inodes with nlink == 0 but still referenced */
 	atomic_long_t s_remove_count;
 
-	/* Pending fsnotify inode refs */
-	atomic_long_t s_fsnotify_inode_refs;
+	/* Number of inode/mount/sb objects that are being watched */
+	atomic_long_t s_fsnotify_connectors;

I've realized inode watches will be double-accounted because we increment s_fsnotify_connectors both when attaching a connector and when grabbing inode reference. It doesn't really matter and avoiding this would require special treatment of inode connectors (we need to decrement s_fsnotify_connectors only after dropping inode reference). So I'll just reflect this in the comment here so that we don't forget.

Honza

Signature · 2 lines
Jan Kara <[email protected]>
SUSE Labs, CR
author reviewer
#6 lore raw

Re: [PATCH v2 3/4] fsnotify: count all objects with attached connectors

Quoted 21 lines · Jan Kara
Quoted 8 lines · Amir Goldstein

Rename s_fsnotify_inode_refs to s_fsnotify_connectors and count all objects with attached connectors, not only inodes with attached connectors.

This will be used to optimize fsnotify() calls on sb without any type of marks.

... just a minor nit below ...

Quoted 8 lines
include/linux/fs.h +0 -0
index 640574294216..d48d2018dfa4 100644
@@ -1507,8 +1507,8 @@ struct super_block {
      /* Number of inodes with nlink == 0 but still referenced */
      atomic_long_t s_remove_count;
 
Quoted 11 lines

include/linux/fs.h struct super_block { · line 1511

Quoted 4 lines
include/linux/fs.h +2 -2
-     /* Pending fsnotify inode refs */
-     atomic_long_t s_fsnotify_inode_refs;
+     /* Number of inode/mount/sb objects that are being watched */
+     atomic_long_t s_fsnotify_connectors;

I've realized inode watches will be double-accounted because we increment s_fsnotify_connectors both when attaching a connector and when grabbing inode reference. It doesn't really matter and avoiding this would require special treatment of inode connectors (we need to decrement s_fsnotify_connectors only after dropping inode reference). So I'll just reflect this in the comment here so that we don't forget.

Great!
I actually have a patch to make the inode reference optional per backend/mark [1], so keeping double account makes things a lot easier for that code as well.

Thanks,
Amir.

[1] https://github.com/amir73il/linux/commits/fsnotify-volatile

author reviewer
#7 lore raw

PATCH v2 4/4 fsnotify: optimize the case of no marks of any type

In reply to Amir Goldstein

Add a simple check in the inline helpers to avoid calling fsnotify() and __fsnotify_parent() in case there are no marks of any type (inode/sb/mount) for an inode's sb, so there can be no objects of any type interested in the event.

---
 include/linux/fsnotify.h | 9 +++++++++
 1 file changed, 9 insertions(+)
include/linux/fsnotify.h +9 -0
index f8acddcf54fb..12d3a7d308ab 100644
@@ -30,6 +30,9 @@ static inline void fsnotify_name(struct inode *dir, __u32 mask,
 				 struct inode *child,
 				 const struct qstr *name, u32 cookie)
 {
+	if (atomic_long_read(&dir->i_sb->s_fsnotify_connectors) == 0)
+		return;
+
 	fsnotify(mask, child, FSNOTIFY_EVENT_INODE, dir, name, NULL, cookie);
 }
 
@@ -41,6 +44,9 @@ static inline void fsnotify_dirent(struct inode *dir, struct dentry *dentry,
 
 static inline void fsnotify_inode(struct inode *inode, __u32 mask)
 {
+	if (atomic_long_read(&inode->i_sb->s_fsnotify_connectors) == 0)
+		return;
+
 	if (S_ISDIR(inode->i_mode))
 		mask |= FS_ISDIR;
 
@@ -53,6 +59,9 @@ static inline int fsnotify_parent(struct dentry *dentry, __u32 mask,
 {
 	struct inode *inode = d_inode(dentry);
 
+	if (atomic_long_read(&inode->i_sb->s_fsnotify_connectors) == 0)
+		return 0;
+
 	if (S_ISDIR(inode->i_mode)) {
 		mask |= FS_ISDIR;
 
Signature · 1 line
2.32.0
maintainer
#8 lore raw
In reply to Amir Goldstein
Quoted 5 lines · Amir Goldstein

Hi Jan,

Following v2 addresses review comments from v1 [1]

[1] https://lore.kernel.org/linux-fsdevel/[email protected]/

Thanks for the patches and for the review to Matthew. I've picked up the patches to my tree.

Honza

Quoted 23 lines

Changes since v1:
- Rebase on 5.14-rc5 + pidfd patches
- Added RVB
- Helper to get connector's sb (Matthew)
- Fix deadlock bug on umount (Jan)

Amir Goldstein (4):
fsnotify: replace igrab() with ihold() on attach connector
fsnotify: count s_fsnotify_inode_refs for attached connectors
fsnotify: count all objects with attached connectors
fsnotify: optimize the case of no marks of any type

 fs/notify/fsnotify.c     |  6 ++---
 fs/notify/fsnotify.h     | 15 ++++++++++++
 fs/notify/mark.c         | 52 ++++++++++++++++++++++++++++++----------
 include/linux/fs.h       |  4 ++--
 include/linux/fsnotify.h |  9 +++++++
 5 files changed, 69 insertions(+), 17 deletions(-)
Signature · 1 line
2.32.0
Signature · 2 lines
Jan Kara <[email protected]>
SUSE Labs, CR