vfs: Turn the unlinked inode counter into a percpu counter
Subsystems (2)
PATCH 1/1 vfs: Turn the unlinked inode counter into a percpu counter
From: Andi Kleen <[email protected]>
The unlinked open inodes super block counter that was added some time ago unfortunately becomes a very hot cache line in some delete heavy high IOPs or tmpfs workloads with enough cores, when files are frequently removed. With TSX it also causes plenty of aborts.
Turn the atomic into a percpu_counter. It's nearly perfectly suited for a percpu counter because it is only checked in very slow paths, like remount.
Original commit:
commit 7ada4db88634429f4da690ad1c4eb73c93085f0c
Author: Miklos Szeredi <[email protected]> Date: Mon Nov 21 12:11:32 2011 +0100
vfs: count unlinked inodes
Add a new counter to the superblock that keeps track of unlinked but not yet deleted inodes.
--- fs/ext2/super.c | 2 +- fs/inode.c | 14 ++++++-------- fs/namespace.c | 4 ++-- fs/super.c | 3 +++ include/linux/fs.h | 2 +- 5 files changed, 13 insertions(+), 12 deletions(-)
fs/ext2/super.c +1 -1
@@ -1218,7 +1218,7 @@ static int ext2_freeze(struct super_block *sb)* because we have unattached inodes and thus filesystem is not fully * consistent. */ -if (atomic_long_read (&sb->s_remove_count)) { +if (percpu_counter_sum (&sb->s_remove_counters)) {ext2_sync_fs (sb,1 );return 0 ; }
fs/inode.c +6 -8
@@ -234,10 +234,8 @@ void __destroy_inode(struct inode *inode)BUG_ON (inode_has_buffers (inode));security_inode_free (inode);fsnotify_inode_delete (inode); -if (!inode->i_nlink) { -WARN_ON (atomic_long_read (&inode->i_sb->s_remove_count) ==0 ); -atomic_long_dec (&inode->i_sb->s_remove_count); - } +if (!inode->i_nlink) +percpu_counter_dec (&inode->i_sb->s_remove_counters);#ifdef CONFIG_FS_POSIX_ACL if (inode->i_acl && inode->i_acl !=ACL_NOT_CACHED )
@@ -281,7 +279,7 @@ void drop_nlink(struct inode *inode)WARN_ON (inode->i_nlink ==0 ); inode->__i_nlink--;if (!inode->i_nlink) -atomic_long_inc (&inode->i_sb->s_remove_count); +percpu_counter_inc (&inode->i_sb->s_remove_counters); }EXPORT_SYMBOL (drop_nlink);
@@ -297,7 +295,7 @@ void clear_nlink(struct inode *inode) {if (inode->i_nlink) { inode->__i_nlink =0 ; -atomic_long_inc (&inode->i_sb->s_remove_count); +percpu_counter_inc (&inode->i_sb->s_remove_counters); } }EXPORT_SYMBOL (clear_nlink);
@@ -317,7 +315,7 @@ void set_nlink(struct inode *inode, unsigned int nlink) }else {/* Yes, some filesystems do change nlink from zero to one */ if (inode->i_nlink ==0 ) -atomic_long_dec (&inode->i_sb->s_remove_count); +percpu_counter_dec (&inode->i_sb->s_remove_counters); inode->__i_nlink = nlink; }
@@ -336,7 +334,7 @@ void inc_nlink(struct inode *inode) {if (unlikely (inode->i_nlink ==0 )) {WARN_ON (!(inode->i_state &I_LINKABLE )); -atomic_long_dec (&inode->i_sb->s_remove_count); +percpu_counter_dec (&inode->i_sb->s_remove_counters); } inode->__i_nlink++;
fs/namespace.c +2 -2
@@ -535,7 +535,7 @@ int sb_prepare_remount_readonly(struct super_block *sb)int err =0 ;/* Racy optimization. Recheck the counter under MNT_WRITE_HOLD */ -if (atomic_long_read (&sb->s_remove_count)) +if (percpu_counter_sum (&sb->s_remove_counters) !=0 )return -EBUSY ;lock_mount_hash ();
@@ -549,7 +549,7 @@ int sb_prepare_remount_readonly(struct super_block *sb) } } } -if (!err &&atomic_long_read (&sb->s_remove_count)) +if (!err &&percpu_counter_sum (&sb->s_remove_counters) !=0 ) err = -EBUSY ;if (!err) {
fs/super.c +3 -0
@@ -142,6 +142,7 @@ static void destroy_super(struct super_block *s)list_lru_destroy (&s->s_inode_lru);for (i =0 ; i <SB_FREEZE_LEVELS ; i++)percpu_counter_destroy (&s->s_writers.counter[i]); +percpu_counter_destroy (&s->s_remove_counters);security_sb_free (s);WARN_ON (!list_empty (&s->s_mounts));kfree (s->s_subtype);
@@ -171,6 +172,8 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags)if (security_sb_alloc (s)) goto fail; +if (percpu_counter_init (&s->s_remove_counters,0 ) <0 ) + goto fail;for (i =0 ; i <SB_FREEZE_LEVELS ; i++) {if (percpu_counter_init (&s->s_writers.counter[i],0 ) <0 ) goto fail;
include/linux/fs.h +1 -1
@@ -1240,7 +1240,7 @@ struct super_block {struct shrinker s_shrink;/* per-sb shrinker handle */ /* Number of inodes with nlink == 0 but still referenced */ -atomic_long_t s_remove_count; +struct percpu_counter s_remove_counters;/* Being remounted read-only */ int s_readonly_remount;
Signature · 1 line
1.9.3
Quoted 58 lines · Andi Kleen
Date: Fri, 11 Jul 2014 13:23:39 -0700
From: Andi Kleen <[email protected]>
To: [email protected]
Cc: [email protected], Andi Kleen <[email protected]>,
Al Viro <[email protected]>, Miklos Szeredi <[email protected]>
Subject: [PATCH] vfs: Turn the unlinked inode counter into a percpu counterFrom: Andi Kleen <[email protected]>
The unlinked open inodes super block counter that was added some time ago unfortunately becomes a very hot cache line in some delete heavy high IOPs or tmpfs workloads with enough cores, when files are frequently removed. With TSX it also causes plenty of aborts.
Turn the atomic into a
percpu_counter. It's nearly perfectly suited for a percpu counter because it is only checked in very slow paths, like remount.Original commit:
commit
7ada4db88634429f4da690ad1c4eb73c93085f0cAuthor: Miklos Szeredi <[email protected]> Date: Mon Nov 21 12:11:32 2011 +0100vfs: count unlinked inodes
Add a new counter to the superblock that keeps track of unlinked but not yet deleted inodes.
Cc Al Viro <[email protected]>Cc Miklos Szeredi <[email protected]>Signed-off-by Andi Kleen <[email protected]>--- fs/ext2/super.c | 2 +- fs/inode.c | 14 ++++++-------- fs/namespace.c | 4 ++-- fs/super.c | 3 +++ include/linux/fs.h | 2 +- 5 files changed, 13 insertions(+), 12 deletions(-)fs/ext2/super.c +1
-1@@ -1218,7 +1218,7 @@ static int ext2_freeze(struct super_block *sb)* because we have unattached inodes and thus filesystem is not fully * consistent. */ -if (atomic_long_read (&sb->s_remove_count)) { +if (percpu_counter_sum (&sb->s_remove_counters)) {ext2_sync_fs (sb,1 );return 0 ; }fs/inode.c +0
-0
fs/inode.c void __destroy_inode(struct inode *inode) · line 238
Quoted 5 lines
fs/inode.c +0
-2@@ -234,10 +234,8 @@ void __destroy_inode(struct inode *inode)BUG_ON (inode_has_buffers (inode));security_inode_free (inode);fsnotify_inode_delete (inode); -if (!inode->i_nlink) { -WARN_ON (atomic_long_read (&inode->i_sb->s_remove_count) ==0 );
This is not mentioned in the description, but I guess it makes sense since it'd be quite expensive.
-Lukas
Quoted 100 lines
fs/inode.c +6
-6-atomic_long_dec (&inode->i_sb->s_remove_count); - } +if (!inode->i_nlink) +percpu_counter_dec (&inode->i_sb->s_remove_counters);#ifdef CONFIG_FS_POSIX_ACL if (inode->i_acl && inode->i_acl !=ACL_NOT_CACHED )@@ -281,7 +279,7 @@ void drop_nlink(struct inode *inode)WARN_ON (inode->i_nlink ==0 ); inode->__i_nlink--;if (!inode->i_nlink) -atomic_long_inc (&inode->i_sb->s_remove_count); +percpu_counter_inc (&inode->i_sb->s_remove_counters); }EXPORT_SYMBOL (drop_nlink);@@ -297,7 +295,7 @@ void clear_nlink(struct inode *inode) {if (inode->i_nlink) { inode->__i_nlink =0 ; -atomic_long_inc (&inode->i_sb->s_remove_count); +percpu_counter_inc (&inode->i_sb->s_remove_counters); } }EXPORT_SYMBOL (clear_nlink);@@ -317,7 +315,7 @@ void set_nlink(struct inode *inode, unsigned int nlink) }else {/* Yes, some filesystems do change nlink from zero to one */ if (inode->i_nlink ==0 ) -atomic_long_dec (&inode->i_sb->s_remove_count); +percpu_counter_dec (&inode->i_sb->s_remove_counters); inode->__i_nlink = nlink; }@@ -336,7 +334,7 @@ void inc_nlink(struct inode *inode) {if (unlikely (inode->i_nlink ==0 )) {WARN_ON (!(inode->i_state &I_LINKABLE )); -atomic_long_dec (&inode->i_sb->s_remove_count); +percpu_counter_dec (&inode->i_sb->s_remove_counters); } inode->__i_nlink++;fs/namespace.c +2
-2@@ -535,7 +535,7 @@ int sb_prepare_remount_readonly(struct super_block *sb)int err =0 ;/* Racy optimization. Recheck the counter under MNT_WRITE_HOLD */ -if (atomic_long_read (&sb->s_remove_count)) +if (percpu_counter_sum (&sb->s_remove_counters) !=0 )return -EBUSY ;lock_mount_hash ();@@ -549,7 +549,7 @@ int sb_prepare_remount_readonly(struct super_block *sb) } } } -if (!err &&atomic_long_read (&sb->s_remove_count)) +if (!err &&percpu_counter_sum (&sb->s_remove_counters) !=0 ) err = -EBUSY ;if (!err) {fs/super.c +3
-0@@ -142,6 +142,7 @@ static void destroy_super(struct super_block *s)list_lru_destroy (&s->s_inode_lru);for (i =0 ; i <SB_FREEZE_LEVELS ; i++)percpu_counter_destroy (&s->s_writers.counter[i]); +percpu_counter_destroy (&s->s_remove_counters);security_sb_free (s);WARN_ON (!list_empty (&s->s_mounts));kfree (s->s_subtype);@@ -171,6 +172,8 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags)if (security_sb_alloc (s)) goto fail; +if (percpu_counter_init (&s->s_remove_counters,0 ) <0 ) + goto fail;for (i =0 ; i <SB_FREEZE_LEVELS ; i++) {if (percpu_counter_init (&s->s_writers.counter[i],0 ) <0 ) goto fail;include/linux/fs.h +1
-1@@ -1240,7 +1240,7 @@ struct super_block {struct shrinker s_shrink;/* per-sb shrinker handle */ /* Number of inodes with nlink == 0 but still referenced */ -atomic_long_t s_remove_count; +struct percpu_counter s_remove_counters;/* Being remounted read-only */ int s_readonly_remount;
Quoted 5 lines
Quoted 5 lines
fs/inode.c +0
-0@@ -234,10 +234,8 @@ void __destroy_inode(struct inode *inode)
Quoted 8 lines
fs/inode.c void __destroy_inode(struct inode *inode) · line 238
Quoted 5 lines
fs/inode.c +0
-2BUG_ON (inode_has_buffers (inode));security_inode_free (inode);fsnotify_inode_delete (inode); -if (!inode->i_nlink) { -WARN_ON (atomic_long_read (&inode->i_sb->s_remove_count) ==0 );This is not mentioned in the description, but I guess it makes sense since it'd be quite expensive.
Yes. I guess we could still check the local counter. I can re-add that.
-Andi
Quoted 8 lines · Andi Kleen
Quoted 6 lines
fsnotify_inode_delete(inode);fs/inode.c void __destroy_inode(struct inode *inode) · line 238
Quoted 2 lines
fs/inode.c +0
-2-if (!inode->i_nlink) { -WARN_ON (atomic_long_read (&inode->i_sb->s_remove_count) ==0 );This is not mentioned in the description, but I guess it makes sense since it'd be quite expensive.
Yes. I guess we could still check the local counter. I can re-add that.
Actually it's not possible, because the original count could be on another CPU.
-Andi
Signature · 1 line
[email protected] -- Speaking for myself only