Skip to contentSkip to conversation

vfs: Turn the unlinked inode counter into a percpu counter

PATCH In review Andi Kleen 3 participants 4 messages Jul 11, 2014 – Jul 14, 2014

Subsystems (2)
lore
Share & tools
lore

Andi Kleen

author
#0 lore raw

PATCH 1/1 vfs: Turn the unlinked inode counter into a percpu counter

From: Andi Kleen <[email protected]>

The unlinked open inodes super block counter that was added some time ago unfortunately becomes a very hot cache line in some delete heavy high IOPs or tmpfs workloads with enough cores, when files are frequently removed. With TSX it also causes plenty of aborts.

Turn the atomic into a percpu_counter. It's nearly perfectly suited for a percpu counter because it is only checked in very slow paths, like remount.

Original commit:

commit 7ada4db88634429f4da690ad1c4eb73c93085f0c

Author: Miklos Szeredi <[email protected]>
Date:   Mon Nov 21 12:11:32 2011 +0100

vfs: count unlinked inodes

Add a new counter to the superblock that keeps track of unlinked but not yet deleted inodes.

Cc Al Viro <[email protected]>
Cc Miklos Szeredi <[email protected]>
---
 fs/ext2/super.c    |  2 +-
 fs/inode.c         | 14 ++++++--------
 fs/namespace.c     |  4 ++--
 fs/super.c         |  3 +++
 include/linux/fs.h |  2 +-
 5 files changed, 13 insertions(+), 12 deletions(-)
fs/ext2/super.c +1 -1
index 3750031..ac0ad9a 100644
@@ -1218,7 +1218,7 @@ static int ext2_freeze(struct super_block *sb)
 	 * because we have unattached inodes and thus filesystem is not fully
 	 * consistent.
 	 */
-	if (atomic_long_read(&sb->s_remove_count)) {
+	if (percpu_counter_sum(&sb->s_remove_counters)) {
 		ext2_sync_fs(sb, 1);
 		return 0;
 	}
fs/inode.c +6 -8
index f96d2a6..4820c35 100644
@@ -234,10 +234,8 @@ void __destroy_inode(struct inode *inode)
 	BUG_ON(inode_has_buffers(inode));
 	security_inode_free(inode);
 	fsnotify_inode_delete(inode);
-	if (!inode->i_nlink) {
-		WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0);
-		atomic_long_dec(&inode->i_sb->s_remove_count);
-	}
+	if (!inode->i_nlink)
+		percpu_counter_dec(&inode->i_sb->s_remove_counters);
 
 #ifdef CONFIG_FS_POSIX_ACL
 	if (inode->i_acl && inode->i_acl != ACL_NOT_CACHED)
@@ -281,7 +279,7 @@ void drop_nlink(struct inode *inode)
 	WARN_ON(inode->i_nlink == 0);
 	inode->__i_nlink--;
 	if (!inode->i_nlink)
-		atomic_long_inc(&inode->i_sb->s_remove_count);
+		percpu_counter_inc(&inode->i_sb->s_remove_counters);
 }
 EXPORT_SYMBOL(drop_nlink);
 
@@ -297,7 +295,7 @@ void clear_nlink(struct inode *inode)
 {
 	if (inode->i_nlink) {
 		inode->__i_nlink = 0;
-		atomic_long_inc(&inode->i_sb->s_remove_count);
+		percpu_counter_inc(&inode->i_sb->s_remove_counters);
 	}
 }
 EXPORT_SYMBOL(clear_nlink);
@@ -317,7 +315,7 @@ void set_nlink(struct inode *inode, unsigned int nlink)
 	} else {
 		/* Yes, some filesystems do change nlink from zero to one */
 		if (inode->i_nlink == 0)
-			atomic_long_dec(&inode->i_sb->s_remove_count);
+			percpu_counter_dec(&inode->i_sb->s_remove_counters);
 
 		inode->__i_nlink = nlink;
 	}
@@ -336,7 +334,7 @@ void inc_nlink(struct inode *inode)
 {
 	if (unlikely(inode->i_nlink == 0)) {
 		WARN_ON(!(inode->i_state & I_LINKABLE));
-		atomic_long_dec(&inode->i_sb->s_remove_count);
+		percpu_counter_dec(&inode->i_sb->s_remove_counters);
 	}
 
 	inode->__i_nlink++;
fs/namespace.c +2 -2
index 182bc41..20d33d9 100644
@@ -535,7 +535,7 @@ int sb_prepare_remount_readonly(struct super_block *sb)
 	int err = 0;
 
 	/* Racy optimization.  Recheck the counter under MNT_WRITE_HOLD */
-	if (atomic_long_read(&sb->s_remove_count))
+	if (percpu_counter_sum(&sb->s_remove_counters) != 0)
 		return -EBUSY;
 
 	lock_mount_hash();
@@ -549,7 +549,7 @@ int sb_prepare_remount_readonly(struct super_block *sb)
 			}
 		}
 	}
-	if (!err && atomic_long_read(&sb->s_remove_count))
+	if (!err && percpu_counter_sum(&sb->s_remove_counters) != 0)
 		err = -EBUSY;
 
 	if (!err) {
fs/super.c +3 -0
index 48377f7..b7a8a45 100644
@@ -142,6 +142,7 @@ static void destroy_super(struct super_block *s)
 	list_lru_destroy(&s->s_inode_lru);
 	for (i = 0; i < SB_FREEZE_LEVELS; i++)
 		percpu_counter_destroy(&s->s_writers.counter[i]);
+	percpu_counter_destroy(&s->s_remove_counters);
 	security_sb_free(s);
 	WARN_ON(!list_empty(&s->s_mounts));
 	kfree(s->s_subtype);
@@ -171,6 +172,8 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags)
 	if (security_sb_alloc(s))
 		goto fail;
 
+	if (percpu_counter_init(&s->s_remove_counters, 0) < 0)
+		goto fail;
 	for (i = 0; i < SB_FREEZE_LEVELS; i++) {
 		if (percpu_counter_init(&s->s_writers.counter[i], 0) < 0)
 			goto fail;
include/linux/fs.h +1 -1
index 8780312..0374552 100644
@@ -1240,7 +1240,7 @@ struct super_block {
 	struct shrinker s_shrink;	/* per-sb shrinker handle */
 
 	/* Number of inodes with nlink == 0 but still referenced */
-	atomic_long_t s_remove_count;
+	struct percpu_counter s_remove_counters;
 
 	/* Being remounted read-only */
 	int s_readonly_remount;
Signature · 1 line
1.9.3
#1 lore raw
Quoted 58 lines · Andi Kleen

Date: Fri, 11 Jul 2014 13:23:39 -0700
From: Andi Kleen <[email protected]>
To: [email protected]
Cc: [email protected], Andi Kleen <[email protected]>,
Al Viro <[email protected]>, Miklos Szeredi <[email protected]>
Subject: [PATCH] vfs: Turn the unlinked inode counter into a percpu counter

From: Andi Kleen <[email protected]>

The unlinked open inodes super block counter that was added some time ago unfortunately becomes a very hot cache line in some delete heavy high IOPs or tmpfs workloads with enough cores, when files are frequently removed. With TSX it also causes plenty of aborts.

Turn the atomic into a percpu_counter. It's nearly perfectly suited for a percpu counter because it is only checked in very slow paths, like remount.

Original commit:

commit 7ada4db88634429f4da690ad1c4eb73c93085f0c

Author: Miklos Szeredi <[email protected]>
Date:   Mon Nov 21 12:11:32 2011 +0100

vfs: count unlinked inodes

Add a new counter to the superblock that keeps track of unlinked but not yet deleted inodes.

Cc Al Viro <[email protected]>
Cc Miklos Szeredi <[email protected]>
---
 fs/ext2/super.c    |  2 +-
 fs/inode.c         | 14 ++++++--------
 fs/namespace.c     |  4 ++--
 fs/super.c         |  3 +++
 include/linux/fs.h |  2 +-
 5 files changed, 13 insertions(+), 12 deletions(-)
fs/ext2/super.c +1 -1
index 3750031..ac0ad9a 100644
@@ -1218,7 +1218,7 @@ static int ext2_freeze(struct super_block *sb)
 	 * because we have unattached inodes and thus filesystem is not fully
 	 * consistent.
 	 */
-	if (atomic_long_read(&sb->s_remove_count)) {
+	if (percpu_counter_sum(&sb->s_remove_counters)) {
 		ext2_sync_fs(sb, 1);
 		return 0;
 	}
fs/inode.c +0 -0
index f96d2a6..4820c35 100644

fs/inode.c void __destroy_inode(struct inode *inode) · line 238

Quoted 5 lines
fs/inode.c +0 -2
@@ -234,10 +234,8 @@ void __destroy_inode(struct inode *inode)
 	BUG_ON(inode_has_buffers(inode));
 	security_inode_free(inode);
 	fsnotify_inode_delete(inode);
-	if (!inode->i_nlink) {
-		WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0);

This is not mentioned in the description, but I guess it makes sense since it'd be quite expensive.

-Lukas

Quoted 100 lines
fs/inode.c +6 -6
-		atomic_long_dec(&inode->i_sb->s_remove_count);
-	}
+	if (!inode->i_nlink)
+		percpu_counter_dec(&inode->i_sb->s_remove_counters);
 
 #ifdef CONFIG_FS_POSIX_ACL
 	if (inode->i_acl && inode->i_acl != ACL_NOT_CACHED)
@@ -281,7 +279,7 @@ void drop_nlink(struct inode *inode)
 	WARN_ON(inode->i_nlink == 0);
 	inode->__i_nlink--;
 	if (!inode->i_nlink)
-		atomic_long_inc(&inode->i_sb->s_remove_count);
+		percpu_counter_inc(&inode->i_sb->s_remove_counters);
 }
 EXPORT_SYMBOL(drop_nlink);
 
@@ -297,7 +295,7 @@ void clear_nlink(struct inode *inode)
 {
 	if (inode->i_nlink) {
 		inode->__i_nlink = 0;
-		atomic_long_inc(&inode->i_sb->s_remove_count);
+		percpu_counter_inc(&inode->i_sb->s_remove_counters);
 	}
 }
 EXPORT_SYMBOL(clear_nlink);
@@ -317,7 +315,7 @@ void set_nlink(struct inode *inode, unsigned int nlink)
 	} else {
 		/* Yes, some filesystems do change nlink from zero to one */
 		if (inode->i_nlink == 0)
-			atomic_long_dec(&inode->i_sb->s_remove_count);
+			percpu_counter_dec(&inode->i_sb->s_remove_counters);
 
 		inode->__i_nlink = nlink;
 	}
@@ -336,7 +334,7 @@ void inc_nlink(struct inode *inode)
 {
 	if (unlikely(inode->i_nlink == 0)) {
 		WARN_ON(!(inode->i_state & I_LINKABLE));
-		atomic_long_dec(&inode->i_sb->s_remove_count);
+		percpu_counter_dec(&inode->i_sb->s_remove_counters);
 	}
 
 	inode->__i_nlink++;
fs/namespace.c +2 -2
index 182bc41..20d33d9 100644
@@ -535,7 +535,7 @@ int sb_prepare_remount_readonly(struct super_block *sb)
 	int err = 0;
 
 	/* Racy optimization.  Recheck the counter under MNT_WRITE_HOLD */
-	if (atomic_long_read(&sb->s_remove_count))
+	if (percpu_counter_sum(&sb->s_remove_counters) != 0)
 		return -EBUSY;
 
 	lock_mount_hash();
@@ -549,7 +549,7 @@ int sb_prepare_remount_readonly(struct super_block *sb)
 			}
 		}
 	}
-	if (!err && atomic_long_read(&sb->s_remove_count))
+	if (!err && percpu_counter_sum(&sb->s_remove_counters) != 0)
 		err = -EBUSY;
 
 	if (!err) {
fs/super.c +3 -0
index 48377f7..b7a8a45 100644
@@ -142,6 +142,7 @@ static void destroy_super(struct super_block *s)
 	list_lru_destroy(&s->s_inode_lru);
 	for (i = 0; i < SB_FREEZE_LEVELS; i++)
 		percpu_counter_destroy(&s->s_writers.counter[i]);
+	percpu_counter_destroy(&s->s_remove_counters);
 	security_sb_free(s);
 	WARN_ON(!list_empty(&s->s_mounts));
 	kfree(s->s_subtype);
@@ -171,6 +172,8 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags)
 	if (security_sb_alloc(s))
 		goto fail;
 
+	if (percpu_counter_init(&s->s_remove_counters, 0) < 0)
+		goto fail;
 	for (i = 0; i < SB_FREEZE_LEVELS; i++) {
 		if (percpu_counter_init(&s->s_writers.counter[i], 0) < 0)
 			goto fail;
include/linux/fs.h +1 -1
index 8780312..0374552 100644
@@ -1240,7 +1240,7 @@ struct super_block {
 	struct shrinker s_shrink;	/* per-sb shrinker handle */
 
 	/* Number of inodes with nlink == 0 but still referenced */
-	atomic_long_t s_remove_count;
+	struct percpu_counter s_remove_counters;
 
 	/* Being remounted read-only */
 	int s_readonly_remount;
#2 lore raw
Quoted 5 lines
Quoted 5 lines
fs/inode.c +0 -0
index f96d2a6..4820c35 100644
@@ -234,10 +234,8 @@ void __destroy_inode(struct inode *inode)
Quoted 8 lines

fs/inode.c void __destroy_inode(struct inode *inode) · line 238

Quoted 5 lines
fs/inode.c +0 -2
 	BUG_ON(inode_has_buffers(inode));
 	security_inode_free(inode);
 	fsnotify_inode_delete(inode);
-	if (!inode->i_nlink) {
-		WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0);

This is not mentioned in the description, but I guess it makes sense since it'd be quite expensive.

Yes. I guess we could still check the local counter. I can re-add that.

-Andi

author
#3 lore raw
Quoted 8 lines · Andi Kleen
Quoted 6 lines

fsnotify_inode_delete(inode);

fs/inode.c void __destroy_inode(struct inode *inode) · line 238

Quoted 2 lines
fs/inode.c +0 -2
-	if (!inode->i_nlink) {
-		WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0);

This is not mentioned in the description, but I guess it makes sense since it'd be quite expensive.

Yes. I guess we could still check the local counter. I can re-add that.

Actually it's not possible, because the original count could be on another CPU.

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only