Skip to contentSkip to conversation

rcu-walk and dcache scaling

PATCH In review Nick Piggin 12 participants 97 messages Nov 27, 2010 – Feb 14, 2011 Page 2 of 2

Subsystems (58)
lore
Share & tools
lore
#71 lore raw

PATCH 39/46 fs: d_revalidate_rcu for rcu-walk

In reply to Nick Piggin

This seems to be the best way to get filesystems aware of RCU walking in their d_revalidate routines. Needs more documentation if it works out OK.

---
 fs/dcache.c            |    2 +
 fs/ecryptfs/dentry.c   |   15 ++++++++----
 fs/fat/namei_vfat.c    |   14 ++++++------
 fs/fuse/dir.c          |    7 ++++-
 fs/namei.c             |   56 ++++++++++++++++++++++++++++++++++-------------
 fs/sysfs/dir.c         |   18 ++++++++++++--
 include/linux/dcache.h |    7 +++++-
 7 files changed, 85 insertions(+), 34 deletions(-)
fs/dcache.c +2 -0
index fa6e7a5..67a08d4 100644
@@ -1243,6 +1243,8 @@ void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
 		dentry->d_flags |= DCACHE_OP_COMPARE;
 	if (op->d_revalidate)
 		dentry->d_flags |= DCACHE_OP_REVALIDATE;
+	if (op->d_revalidate_rcu)
+		dentry->d_flags |= DCACHE_OP_REVALIDATE_RCU;
 	if (op->d_delete)
 		dentry->d_flags |= DCACHE_OP_DELETE;
 
fs/ecryptfs/dentry.c +10 -5
index 906e803..f9a8be8 100644
@@ -30,7 +30,7 @@
 #include "ecryptfs_kernel.h"
 
 /**
- * ecryptfs_d_revalidate - revalidate an ecryptfs dentry
+ * ecryptfs_d_revalidate_rcu - revalidate an ecryptfs dentry
  * @dentry: The ecryptfs dentry
  * @nd: The associated nameidata
  *
@@ -42,7 +42,7 @@
  * Returns 1 if valid, 0 otherwise.
  *
  */
-static int ecryptfs_d_revalidate(struct dentry *dentry, struct nameidata *nd)
+static int ecryptfs_d_revalidate_rcu(struct dentry *dentry, struct nameidata *nd)
 {
 	struct dentry *lower_dentry = ecryptfs_dentry_to_lower(dentry);
 	struct vfsmount *lower_mnt = ecryptfs_dentry_to_lower_mnt(dentry);
@@ -50,13 +50,18 @@ static int ecryptfs_d_revalidate(struct dentry *dentry, struct nameidata *nd)
 	struct vfsmount *vfsmount_save;
 	int rc = 1;
 
-	if (!lower_dentry->d_op || !lower_dentry->d_op->d_revalidate)
+	if (!(lower_dentry->d_flags & DCACHE_OP_REVALIDATE_EITHER))
 		goto out;
+	if (nd->flags & LOOKUP_RCU)
+		return -ECHILD;
 	dentry_save = nd->path.dentry;
 	vfsmount_save = nd->path.mnt;
 	nd->path.dentry = lower_dentry;
 	nd->path.mnt = lower_mnt;
-	rc = lower_dentry->d_op->d_revalidate(lower_dentry, nd);
+	if (lower_dentry->d_flags & DCACHE_OP_REVALIDATE)
+		rc = lower_dentry->d_op->d_revalidate(lower_dentry, nd);
+	else
+		rc = lower_dentry->d_op->d_revalidate_rcu(lower_dentry, nd);
 	nd->path.dentry = dentry_save;
 	nd->path.mnt = vfsmount_save;
 	if (dentry->d_inode) {
@@ -91,6 +96,6 @@ static void ecryptfs_d_release(struct dentry *dentry)
 }
 
 const struct dentry_operations ecryptfs_dops = {
-	.d_revalidate = ecryptfs_d_revalidate,
+	.d_revalidate_rcu = ecryptfs_d_revalidate_rcu,
 	.d_release = ecryptfs_d_release,
 };
fs/fat/namei_vfat.c +7 -7
index b721715..a169f24 100644
@@ -31,7 +31,7 @@
  * If it happened, the negative dentry isn't actually negative
  * anymore.  So, drop it.
  */
-static int vfat_revalidate_shortname(struct dentry *dentry)
+static int vfat_revalidate_rcu_shortname(struct dentry *dentry)
 {
 	int ret = 1;
 	spin_lock(&dentry->d_lock);
@@ -41,15 +41,15 @@ static int vfat_revalidate_shortname(struct dentry *dentry)
 	return ret;
 }
 
-static int vfat_revalidate(struct dentry *dentry, struct nameidata *nd)
+static int vfat_revalidate_rcu(struct dentry *dentry, struct nameidata *nd)
 {
 	/* This is not negative dentry. Always valid. */
 	if (dentry->d_inode)
 		return 1;
-	return vfat_revalidate_shortname(dentry);
+	return vfat_revalidate_rcu_shortname(dentry);
 }
 
-static int vfat_revalidate_ci(struct dentry *dentry, struct nameidata *nd)
+static int vfat_revalidate_rcu_ci(struct dentry *dentry, struct nameidata *nd)
 {
 	/*
 	 * This is not negative dentry. Always valid.
@@ -81,7 +81,7 @@ static int vfat_revalidate_ci(struct dentry *dentry, struct nameidata *nd)
 			return 0;
 	}
 
-	return vfat_revalidate_shortname(dentry);
+	return vfat_revalidate_rcu_shortname(dentry);
 }
 
 /* returns the length of a struct qstr, ignoring trailing dots */
@@ -175,13 +175,13 @@ static int vfat_cmp(const struct dentry *parent,
 }
 
 static const struct dentry_operations vfat_ci_dentry_ops = {
-	.d_revalidate	= vfat_revalidate_ci,
+	.d_revalidate	= vfat_revalidate_rcu_ci,
 	.d_hash		= vfat_hashi,
 	.d_compare	= vfat_cmpi,
 };
 
 static const struct dentry_operations vfat_dentry_ops = {
-	.d_revalidate	= vfat_revalidate,
+	.d_revalidate	= vfat_revalidate_rcu,
 	.d_hash		= vfat_hash,
 	.d_compare	= vfat_cmp,
 };
fs/fuse/dir.c +5 -2
index c9a8a42..e409185 100644
@@ -154,7 +154,7 @@ u64 fuse_get_attr_version(struct fuse_conn *fc)
  * the lookup once more.  If the lookup results in the same inode,
  * then refresh the attributes, timeouts and mark the dentry valid.
  */
-static int fuse_dentry_revalidate(struct dentry *entry, struct nameidata *nd)
+static int fuse_dentry_revalidate_rcu(struct dentry *entry, struct nameidata *nd)
 {
 	struct inode *inode = entry->d_inode;
 
@@ -169,6 +169,9 @@ static int fuse_dentry_revalidate(struct dentry *entry, struct nameidata *nd)
 		struct dentry *parent;
 		u64 attr_version;
 
+		if (nd->flags & LOOKUP_RCU)
+			return -ECHILD;
+
 		/* For negative dentries, always do a fresh lookup */
 		if (!inode)
 			return 0;
@@ -225,7 +228,7 @@ static int invalid_nodeid(u64 nodeid)
 }
 
 const struct dentry_operations fuse_dentry_operations = {
-	.d_revalidate	= fuse_dentry_revalidate,
+	.d_revalidate_rcu = fuse_dentry_revalidate_rcu,
 };
 
 int fuse_valid_type(int m)
fs/namei.c +40 -16

111 lines not on this page · raw message

fs/sysfs/dir.c +15 -3
index 3e076ca..c3fff57 100644
@@ -237,12 +237,21 @@ static int sysfs_dentry_delete(const struct dentry *dentry)
 	return !!(sd->s_flags & SYSFS_FLAG_REMOVED);
 }
 
-static int sysfs_dentry_revalidate(struct dentry *dentry, struct nameidata *nd)
+static int sysfs_dentry_revalidate_rcu(struct dentry *dentry, struct nameidata *nd)
 {
 	struct sysfs_dirent *sd = dentry->d_fsdata;
 	int is_dir;
 
-	mutex_lock(&sysfs_mutex);
+	if (nd->flags & LOOKUP_RCU) {
+		if (!mutex_trylock(&sysfs_mutex))
+			return -ECHILD;
+		/* ensure dentry still exists, now under the sysfs_mutex */
+		if (read_seqcount_retry(&dentry->d_seq, nd->seq)) {
+			mutex_unlock(&sysfs_mutex);
+			return -ECHILD;
+		}
+	} else
+		mutex_lock(&sysfs_mutex);
 
 	/* The sysfs dirent has been deleted */
 	if (sd->s_flags & SYSFS_FLAG_REMOVED)
@@ -272,6 +281,9 @@ out_bad:
 	 */
 	is_dir = (sysfs_type(sd) == SYSFS_DIR);
 	mutex_unlock(&sysfs_mutex);
+	if (nd->flags & LOOKUP_RCU)
+		return -ECHILD;
+
 	if (is_dir) {
 		/* If we have submounts we must allow the vfs caches
 		 * to lie about the state of the filesystem to prevent
@@ -294,7 +306,7 @@ static void sysfs_dentry_iput(struct dentry *dentry, struct inode *inode)
 }
 
 static const struct dentry_operations sysfs_dentry_ops = {
-	.d_revalidate	= sysfs_dentry_revalidate,
+	.d_revalidate_rcu = sysfs_dentry_revalidate_rcu,
 	.d_delete	= sysfs_dentry_delete,
 	.d_iput		= sysfs_dentry_iput,
 };
include/linux/dcache.h +6 -1
index 3c5cafc..72f5f32 100644
@@ -132,6 +132,7 @@ enum dentry_d_lock_class
 
 struct dentry_operations {
 	int (*d_revalidate)(struct dentry *, struct nameidata *);
+	int (*d_revalidate_rcu)(struct dentry *, struct nameidata *);
 	int (*d_hash)(const struct dentry *, const struct inode *,
 			struct qstr *);
 	int (*d_compare)(const struct dentry *,
@@ -184,8 +185,12 @@ struct dentry_operations {
 #define DCACHE_OP_HASH		0x1000
 #define DCACHE_OP_COMPARE	0x2000
 #define DCACHE_OP_REVALIDATE	0x4000
-#define DCACHE_OP_DELETE	0x8000
+#define DCACHE_OP_REVALIDATE_RCU 0x8000
 
+#define DCACHE_OP_DELETE	0x10000
+
+#define DCACHE_OP_REVALIDATE_EITHER	\
+	(DCACHE_OP_REVALIDATE|DCACHE_OP_REVALIDATE_RCU)
 
 extern spinlock_t dcache_inode_lock;
 extern spinlock_t dcache_hash_lock;
Signature · 1 line
1.7.1
#72 lore raw

PATCH 40/46 fs: provide rcu-walk aware permission i_ops

In reply to Nick Piggin
---
 fs/namei.c         |  131 ++++++++++++++++++++++++++++++++++++++++------------
 include/linux/fs.h |    6 ++
 2 files changed, 107 insertions(+), 30 deletions(-)
fs/namei.c +101 -30
index d8f7ece..7fa6119 100644
@@ -169,8 +169,36 @@ EXPORT_SYMBOL(putname);
 /*
  * This does basic POSIX ACL permission checking
  */
-static inline int __acl_permission_check(struct inode *inode, int mask,
-		int (*check_acl)(struct inode *inode, int mask), int rcu)
+static int acl_permission_check_rcu(struct inode *inode, int mask, unsigned int flags,
+		int (*check_acl_rcu)(struct inode *inode, int mask, unsigned int flags))
+{
+	umode_t			mode = inode->i_mode;
+
+	mask &= MAY_READ | MAY_WRITE | MAY_EXEC;
+
+	if (current_fsuid() == inode->i_uid)
+		mode >>= 6;
+	else {
+		if (IS_POSIXACL(inode) && (mode & S_IRWXG) && check_acl_rcu) {
+			int error = check_acl_rcu(inode, mask, flags);
+			if (error != -EAGAIN)
+				return error;
+		}
+
+		if (in_group_p(inode->i_gid))
+			mode >>= 3;
+	}
+
+	/*
+	 * If the DACs are ok we don't need any capability check.
+	 */
+	if ((mask & ~mode) == 0)
+		return 0;
+	return -EACCES;
+}
+
+static int acl_permission_check(struct inode *inode, int mask, unsigned int flags,
+		int (*check_acl)(struct inode *inode, int mask))
 {
 	umode_t			mode = inode->i_mode;
 
@@ -180,7 +208,7 @@ static inline int __acl_permission_check(struct inode *inode, int mask,
 		mode >>= 6;
 	else {
 		if (IS_POSIXACL(inode) && (mode & S_IRWXG) && check_acl) {
-			if (rcu) {
+			if (flags) {
 				return -ECHILD;
 			} else {
 				int error = check_acl(inode, mask);
@@ -201,10 +229,52 @@ static inline int __acl_permission_check(struct inode *inode, int mask,
 	return -EACCES;
 }
 
-static inline int acl_permission_check(struct inode *inode, int mask,
-		int (*check_acl)(struct inode *inode, int mask))
+/**
+ * generic_permission_rcu  -  check for access rights on a Posix-like filesystem
+ * @inode:	inode to check access rights for
+ * @mask:	right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC)
+ * @check_acl_rcu: optional callback to check for Posix ACLs
+ * @flags	IPERM_FLAG_ flags.
+ *
+ * Used to check for read/write/execute permissions on a file.
+ * We use "fsuid" for this, letting us set arbitrary permissions
+ * for filesystem access without changing the "normal" uids which
+ * are used for other things.
+ *
+ * generic_permission_rcu must be rcu-walk aware. It should return
+ * -ECHILD in case an rcu-walk request cannot be satisfied (eg.
+ * requires blocking or too much thought!). It would then be called
+ * again in ref-walk mode.
+ */
+int generic_permission_rcu(struct inode *inode, int mask, unsigned int flags,
+	int (*check_acl_rcu)(struct inode *inode, int mask, unsigned int flags))
 {
-	return __acl_permission_check(inode, mask, check_acl, 0);
+	int ret;
+
+	/*
+	 * Do the basic POSIX ACL permission checks.
+	 */
+	ret = acl_permission_check_rcu(inode, mask, flags, check_acl_rcu);
+	if (ret != -EACCES)
+		return ret;
+
+	/*
+	 * Read/write DACs are always overridable.
+	 * Executable DACs are overridable if at least one exec bit is set.
+	 */
+	if (!(mask & MAY_EXEC) || execute_ok(inode))
+		if (capable(CAP_DAC_OVERRIDE))
+			return 0;
+
+	/*
+	 * Searching includes executable on directories, else just read.
+	 */
+	mask &= MAY_READ | MAY_WRITE | MAY_EXEC;
+	if (mask == MAY_READ || (S_ISDIR(inode->i_mode) && !(mask & MAY_WRITE)))
+		if (capable(CAP_DAC_READ_SEARCH))
+			return 0;
+
+	return -EACCES;
 }
 
 /**
@@ -226,7 +296,7 @@ int generic_permission(struct inode *inode, int mask,
 	/*
 	 * Do the basic POSIX ACL permission checks.
 	 */
-	ret = acl_permission_check(inode, mask, check_acl);
+	ret = acl_permission_check(inode, mask, 0, check_acl);
 	if (ret != -EACCES)
 		return ret;
 
@@ -282,8 +352,14 @@ int inode_permission(struct inode *inode, int mask)
 
 	if (inode->i_op->permission)
 		retval = inode->i_op->permission(inode, mask);
+	else if (inode->i_op->permission_rcu)
+		retval = inode->i_op->permission_rcu(inode, mask, 0);
+	else if (inode->i_op->check_acl_rcu)
+		retval = generic_permission_rcu(inode, mask, 0,
+				inode->i_op->check_acl_rcu);
 	else
-		retval = generic_permission(inode, mask, inode->i_op->check_acl);
+		retval = generic_permission(inode, mask,
+				inode->i_op->check_acl);
 
 	if (retval)
 		return retval;
@@ -622,22 +698,26 @@ force_reval_path(struct path *path, struct nameidata *nd)
  * short-cut DAC fails, then call ->permission() to do more
  * complete permission check.
  */
-static inline int __exec_permission(struct inode *inode, int rcu)
+static inline int exec_permission(struct inode *inode, unsigned int flags)
 {
 	int ret;
 
-	if (inode->i_op->permission) {
-		if (rcu)
+	if (inode->i_op->permission_rcu) {
+		ret = inode->i_op->permission_rcu(inode, MAY_EXEC, flags);
+	} else if (inode->i_op->permission) {
+		if (flags)
 			return -ECHILD;
 		ret = inode->i_op->permission(inode, MAY_EXEC);
-		if (!ret)
-			goto ok;
-		return ret;
+	} else if (inode->i_op->check_acl_rcu) {
+		ret = acl_permission_check_rcu(inode, MAY_EXEC, flags,
+				inode->i_op->check_acl_rcu);
+	} else {
+		ret = acl_permission_check(inode, MAY_EXEC, flags,
+				inode->i_op->check_acl);
 	}
-	ret = __acl_permission_check(inode, MAY_EXEC, inode->i_op->check_acl, rcu);
-	if (!ret)
+	if (likely(!ret))
 		goto ok;
-	if (rcu && ret == -ECHILD)
+	if (ret == -ECHILD)
 		return ret;
 
 	if (capable(CAP_DAC_OVERRIDE) || capable(CAP_DAC_READ_SEARCH))
@@ -648,16 +728,6 @@ ok:
 	return security_inode_permission(inode, MAY_EXEC); /* XXX: ok for RCU? */
 }
 
-static int exec_permission(struct inode *inode)
-{
-	return __exec_permission(inode, 0);
-}
-
-static int exec_permission_rcu(struct inode *inode)
-{
-	return __exec_permission(inode, 1);
-}
-
 static __always_inline void set_root(struct nameidata *nd)
 {
 	if (!nd->root.mnt)
@@ -1126,7 +1196,7 @@ static int link_path_walk(const char *name, struct nameidata *nd)
 
 		nd->flags |= LOOKUP_CONTINUE;
 		if (nd->flags & LOOKUP_RCU) {
-			err = exec_permission_rcu(nd->inode);
+			err = exec_permission(nd->inode, IPERM_FLAG_RCU);
 			if (err == -ECHILD) {
 				if (nameidata_drop_rcu(nd))
 					return -ECHILD;
@@ -1134,7 +1204,7 @@ static int link_path_walk(const char *name, struct nameidata *nd)
 			}
 		} else {
 exec_again:
-			err = exec_permission(nd->inode);
+			err = exec_permission(nd->inode, 0);
 		}
  		if (err)
 			break;
@@ -1567,7 +1637,7 @@ static struct dentry *__lookup_hash(struct qstr *name,
 	struct dentry *dentry;
 	int err;
 
-	err = exec_permission(inode);
+	err = exec_permission(inode, 0);
 	if (err)
 		return ERR_PTR(err);
 
@@ -3352,6 +3422,7 @@ EXPORT_SYMBOL(vfs_follow_link);
 EXPORT_SYMBOL(vfs_link);
 EXPORT_SYMBOL(vfs_mkdir);
 EXPORT_SYMBOL(vfs_mknod);
+EXPORT_SYMBOL(generic_permission_rcu);
 EXPORT_SYMBOL(generic_permission);
 EXPORT_SYMBOL(vfs_readlink);
 EXPORT_SYMBOL(vfs_rename);
include/linux/fs.h +6 -0
index 07e8a50..490eedd 100644
@@ -1549,11 +1549,15 @@ struct file_operations {
 	int (*setlease)(struct file *, long, struct file_lock **);
 };
 
+#define IPERM_FLAG_RCU	0x0001
+
 struct inode_operations {
 	struct dentry * (*lookup) (struct inode *,struct dentry *, struct nameidata *);
 	void * (*follow_link) (struct dentry *, struct nameidata *);
 	int (*permission) (struct inode *, int);
+	int (*permission_rcu) (struct inode *, int, unsigned int);
 	int (*check_acl)(struct inode *, int);
+	int (*check_acl_rcu)(struct inode *, int, unsigned int);
 
 	int (*readlink) (struct dentry *, char __user *,int);
 	void (*put_link) (struct dentry *, struct nameidata *, void *);
@@ -2164,6 +2168,8 @@ extern sector_t bmap(struct inode *, sector_t);
 #endif
 extern int notify_change(struct dentry *, struct iattr *);
 extern int inode_permission(struct inode *, int);
+extern int generic_permission_rcu(struct inode *, int, unsigned int,
+		int (*check_acl_rcu)(struct inode *, int, unsigned int));
 extern int generic_permission(struct inode *, int,
 		int (*check_acl)(struct inode *, int));
 
Signature · 1 line
1.7.1
#73 lore raw

PATCH 41/46 fs: provide simple rcu-walk ACL implementation

In reply to Nick Piggin

This simple implementation just checks for no ACLs on the inode, and if so, then the rcu-walk may proceed, otherwise fail it.

This could easily be extended to put acls under RCU and check them under seqlock, if need be. But this implementation is enough to show the rcu-walk aware permissions code for path lookups is working, and will handle cases where there are no ACLs or ACLs in just the final element.

Convert tmpfs, ext2, btrfs. Each of these uses acl/permission code in a different way, so convert them all to provide templates and proof of concept.

---
 fs/btrfs/acl.c              |   19 ++++++++++++-------
 fs/btrfs/ctree.h            |    4 ++--
 fs/btrfs/inode.c            |   14 +++++++-------
 fs/ext2/acl.c               |   11 +++++++++--
 fs/ext2/acl.h               |    8 ++++----
 fs/ext2/file.c              |    2 +-
 fs/ext2/namei.c             |    4 ++--
 fs/ext3/acl.c               |   11 +++++++++--
 fs/ext3/acl.h               |    8 ++++----
 fs/ext3/file.c              |    2 +-
 fs/ext3/namei.c             |    4 ++--
 fs/ext4/acl.c               |   11 +++++++++--
 fs/ext4/acl.h               |    4 ++--
 fs/ext4/file.c              |    2 +-
 fs/ext4/namei.c             |    4 ++--
 fs/generic_acl.c            |   20 ++++++++++++++++++++
 fs/xfs/linux-2.6/xfs_acl.c  |    8 +++++++-
 fs/xfs/linux-2.6/xfs_iops.c |    8 ++++----
 fs/xfs/xfs_acl.h            |    4 ++--
 include/linux/generic_acl.h |    1 +
 include/linux/posix_acl.h   |   19 +++++++++++++++++++
 mm/shmem.c                  |    6 +++---
 22 files changed, 123 insertions(+), 51 deletions(-)
fs/btrfs/acl.c +12 -7
index 2222d16..c03c753 100644
@@ -185,18 +185,23 @@ static int btrfs_xattr_acl_set(struct dentry *dentry, const char *name,
 	return ret;
 }
 
-int btrfs_check_acl(struct inode *inode, int mask)
+int btrfs_check_acl_rcu(struct inode *inode, int mask, unsigned int flags)
 {
 	struct posix_acl *acl;
 	int error = -EAGAIN;
 
-	acl = btrfs_get_acl(inode, ACL_TYPE_ACCESS);
+	if (flags & IPERM_FLAG_RCU) {
+		if (!negative_cached_acl(inode, ACL_TYPE_ACCESS))
+			return -ECHILD;
+	} else {
+		acl = btrfs_get_acl(inode, ACL_TYPE_ACCESS);
 
-	if (IS_ERR(acl))
-		return PTR_ERR(acl);
-	if (acl) {
-		error = posix_acl_permission(inode, acl, mask);
-		posix_acl_release(acl);
+		if (IS_ERR(acl))
+			return PTR_ERR(acl);
+		if (acl) {
+			error = posix_acl_permission(inode, acl, mask);
+			posix_acl_release(acl);
+		}
 	}
 
 	return error;
fs/btrfs/ctree.h +2 -2
index 8db9234..403b069 100644
@@ -2544,9 +2544,9 @@ int btrfs_sync_fs(struct super_block *sb, int wait);
 
 /* acl.c */
 #ifdef CONFIG_BTRFS_FS_POSIX_ACL
-int btrfs_check_acl(struct inode *inode, int mask);
+int btrfs_check_acl_rcu(struct inode *inode, int mask, unsigned int flags);
 #else
-#define btrfs_check_acl NULL
+#define btrfs_check_acl_rcu NULL
 #endif
 int btrfs_init_acl(struct btrfs_trans_handle *trans,
 		   struct inode *inode, struct inode *dir);
fs/btrfs/inode.c +7 -7
index aab3087..c9844e1 100644
@@ -7033,11 +7033,11 @@ static int btrfs_set_page_dirty(struct page *page)
 	return __set_page_dirty_nobuffers(page);
 }
 
-static int btrfs_permission(struct inode *inode, int mask)
+static int btrfs_permission_rcu(struct inode *inode, int mask, unsigned int flags)
 {
 	if ((BTRFS_I(inode)->flags & BTRFS_INODE_READONLY) && (mask & MAY_WRITE))
 		return -EACCES;
-	return generic_permission(inode, mask, btrfs_check_acl);
+	return generic_permission_rcu(inode, mask, flags, btrfs_check_acl_rcu);
 }
 
 static const struct inode_operations btrfs_dir_inode_operations = {
@@ -7056,11 +7056,11 @@ static const struct inode_operations btrfs_dir_inode_operations = {
 	.getxattr	= btrfs_getxattr,
 	.listxattr	= btrfs_listxattr,
 	.removexattr	= btrfs_removexattr,
-	.permission	= btrfs_permission,
+	.permission_rcu	= btrfs_permission_rcu,
 };
 static const struct inode_operations btrfs_dir_ro_inode_operations = {
 	.lookup		= btrfs_lookup,
-	.permission	= btrfs_permission,
+	.permission_rcu	= btrfs_permission_rcu,
 };
 
 static const struct file_operations btrfs_dir_file_operations = {
@@ -7129,14 +7129,14 @@ static const struct inode_operations btrfs_file_inode_operations = {
 	.getxattr	= btrfs_getxattr,
 	.listxattr      = btrfs_listxattr,
 	.removexattr	= btrfs_removexattr,
-	.permission	= btrfs_permission,
+	.permission_rcu	= btrfs_permission_rcu,
 	.fallocate	= btrfs_fallocate,
 	.fiemap		= btrfs_fiemap,
 };
 static const struct inode_operations btrfs_special_inode_operations = {
 	.getattr	= btrfs_getattr,
 	.setattr	= btrfs_setattr,
-	.permission	= btrfs_permission,
+	.permission_rcu	= btrfs_permission_rcu,
 	.setxattr	= btrfs_setxattr,
 	.getxattr	= btrfs_getxattr,
 	.listxattr	= btrfs_listxattr,
@@ -7146,7 +7146,7 @@ static const struct inode_operations btrfs_symlink_inode_operations = {
 	.readlink	= generic_readlink,
 	.follow_link	= page_follow_link_light,
 	.put_link	= page_put_link,
-	.permission	= btrfs_permission,
+	.permission_rcu	= btrfs_permission_rcu,
 	.setxattr	= btrfs_setxattr,
 	.getxattr	= btrfs_getxattr,
 	.listxattr	= btrfs_listxattr,
fs/ext2/acl.c +9 -2
index 2bcc043..43196d7 100644
@@ -232,10 +232,17 @@ ext2_set_acl(struct inode *inode, int type, struct posix_acl *acl)
 }
 
 int
-ext2_check_acl(struct inode *inode, int mask)
+ext2_check_acl_rcu(struct inode *inode, int mask, unsigned int flags)
 {
-	struct posix_acl *acl = ext2_get_acl(inode, ACL_TYPE_ACCESS);
+	struct posix_acl *acl;
+
+	if (flags & IPERM_FLAG_RCU) {
+		if (!negative_cached_acl(inode, ACL_TYPE_ACCESS))
+			return -ECHILD;
+		return -EAGAIN;
+	}
 
+	acl = ext2_get_acl(inode, ACL_TYPE_ACCESS);
 	if (IS_ERR(acl))
 		return PTR_ERR(acl);
 	if (acl) {
fs/ext2/acl.h +4 -4
index 3ff6cbb..fb49393 100644
@@ -54,15 +54,15 @@ static inline int ext2_acl_count(size_t size)
 #ifdef CONFIG_EXT2_FS_POSIX_ACL
 
 /* acl.c */
-extern int ext2_check_acl (struct inode *, int);
+extern int ext2_check_acl_rcu(struct inode *, int, unsigned int);
 extern int ext2_acl_chmod (struct inode *);
 extern int ext2_init_acl (struct inode *, struct inode *);
 
 #else
 #include <linux/sched.h>
-#define ext2_check_acl	NULL
-#define ext2_get_acl	NULL
-#define ext2_set_acl	NULL
+#define ext2_check_acl_rcu	NULL
+#define ext2_get_acl		NULL
+#define ext2_set_acl		NULL
 
 static inline int
 ext2_acl_chmod (struct inode *inode)
fs/ext2/file.c +1 -1
index 49eec94..a34fc5a 100644
@@ -102,6 +102,6 @@ const struct inode_operations ext2_file_inode_operations = {
 	.removexattr	= generic_removexattr,
 #endif
 	.setattr	= ext2_setattr,
-	.check_acl	= ext2_check_acl,
+	.check_acl_rcu	= ext2_check_acl_rcu,
 	.fiemap		= ext2_fiemap,
 };
fs/ext2/namei.c +2 -2
index f8aecd2..5366e02 100644
@@ -417,7 +417,7 @@ const struct inode_operations ext2_dir_inode_operations = {
 	.removexattr	= generic_removexattr,
 #endif
 	.setattr	= ext2_setattr,
-	.check_acl	= ext2_check_acl,
+	.check_acl_rcu	= ext2_check_acl_rcu,
 };
 
 const struct inode_operations ext2_special_inode_operations = {
@@ -428,5 +428,5 @@ const struct inode_operations ext2_special_inode_operations = {
 	.removexattr	= generic_removexattr,
 #endif
 	.setattr	= ext2_setattr,
-	.check_acl	= ext2_check_acl,
+	.check_acl_rcu	= ext2_check_acl_rcu,
 };
fs/ext3/acl.c +9 -2
index 8a11fe2..cff7c13 100644
@@ -240,10 +240,17 @@ ext3_set_acl(handle_t *handle, struct inode *inode, int type,
 }
 
 int
-ext3_check_acl(struct inode *inode, int mask)
+ext3_check_acl_rcu(struct inode *inode, int mask, unsigned int flags)
 {
-	struct posix_acl *acl = ext3_get_acl(inode, ACL_TYPE_ACCESS);
+	struct posix_acl *acl;
+
+	if (flags & IPERM_FLAG_RCU) {
+		if (!negative_cached_acl(inode, ACL_TYPE_ACCESS))
+			return -ECHILD;
+		return -EAGAIN;
+	}
 
+	acl = ext3_get_acl(inode, ACL_TYPE_ACCESS);
 	if (IS_ERR(acl))
 		return PTR_ERR(acl);
 	if (acl) {
fs/ext3/acl.h +4 -4
index 5973346..f2536e6 100644
@@ -54,13 +54,13 @@ static inline int ext3_acl_count(size_t size)
 #ifdef CONFIG_EXT3_FS_POSIX_ACL
 
 /* acl.c */
-extern int ext3_check_acl (struct inode *, int);
-extern int ext3_acl_chmod (struct inode *);
-extern int ext3_init_acl (handle_t *, struct inode *, struct inode *);
+extern int ext3_check_acl_rcu(struct inode *, int, unsigned int);
+extern int ext3_acl_chmod(struct inode *);
+extern int ext3_init_acl(handle_t *, struct inode *, struct inode *);
 
 #else  /* CONFIG_EXT3_FS_POSIX_ACL */
 #include <linux/sched.h>
-#define ext3_check_acl NULL
+#define ext3_check_acl_rcu NULL
 
 static inline int
 ext3_acl_chmod(struct inode *inode)
fs/ext3/file.c +1 -1
index f55df0e..74e5ef7 100644
@@ -79,7 +79,7 @@ const struct inode_operations ext3_file_inode_operations = {
 	.listxattr	= ext3_listxattr,
 	.removexattr	= generic_removexattr,
 #endif
-	.check_acl	= ext3_check_acl,
+	.check_acl_rcu	= ext3_check_acl_rcu,
 	.fiemap		= ext3_fiemap,
 };
 
fs/ext3/namei.c +2 -2
index bce9dce..9b10438 100644
@@ -2464,7 +2464,7 @@ const struct inode_operations ext3_dir_inode_operations = {
 	.listxattr	= ext3_listxattr,
 	.removexattr	= generic_removexattr,
 #endif
-	.check_acl	= ext3_check_acl,
+	.check_acl_rcu	= ext3_check_acl_rcu,
 };
 
 const struct inode_operations ext3_special_inode_operations = {
@@ -2475,5 +2475,5 @@ const struct inode_operations ext3_special_inode_operations = {
 	.listxattr	= ext3_listxattr,
 	.removexattr	= generic_removexattr,
 #endif
-	.check_acl	= ext3_check_acl,
+	.check_acl_rcu	= ext3_check_acl_rcu,
 };
fs/ext4/acl.c +9 -2
index 5e2ed45..ebc7127 100644
@@ -238,10 +238,17 @@ ext4_set_acl(handle_t *handle, struct inode *inode, int type,
 }
 
 int
-ext4_check_acl(struct inode *inode, int mask)
+ext4_check_acl_rcu(struct inode *inode, int mask, unsigned int flags)
 {
-	struct posix_acl *acl = ext4_get_acl(inode, ACL_TYPE_ACCESS);
+	struct posix_acl *acl;
+
+	if (flags & IPERM_FLAG_RCU) {
+		if (!negative_cached_acl(inode, ACL_TYPE_ACCESS))
+			return -ECHILD;
+		return -EAGAIN;
+	}
 
+	acl = ext4_get_acl(inode, ACL_TYPE_ACCESS);
 	if (IS_ERR(acl))
 		return PTR_ERR(acl);
 	if (acl) {
fs/ext4/acl.h +2 -2
index 9d843d5..71e8da7 100644
@@ -54,13 +54,13 @@ static inline int ext4_acl_count(size_t size)
 #ifdef CONFIG_EXT4_FS_POSIX_ACL
 
 /* acl.c */
-extern int ext4_check_acl(struct inode *, int);
+extern int ext4_check_acl_rcu(struct inode *, int, unsigned int);
 extern int ext4_acl_chmod(struct inode *);
 extern int ext4_init_acl(handle_t *, struct inode *, struct inode *);
 
 #else  /* CONFIG_EXT4_FS_POSIX_ACL */
 #include <linux/sched.h>
-#define ext4_check_acl NULL
+#define ext4_check_acl_rcu NULL
 
 static inline int
 ext4_acl_chmod(struct inode *inode)
fs/ext4/file.c +1 -1
index 5a5c55d..f6be0af 100644
@@ -200,7 +200,7 @@ const struct inode_operations ext4_file_inode_operations = {
 	.listxattr	= ext4_listxattr,
 	.removexattr	= generic_removexattr,
 #endif
-	.check_acl	= ext4_check_acl,
+	.check_acl_rcu	= ext4_check_acl_rcu,
 	.fallocate	= ext4_fallocate,
 	.fiemap		= ext4_fiemap,
 };
fs/ext4/namei.c +2 -2
index 92203b8..6145f55 100644
@@ -2511,7 +2511,7 @@ const struct inode_operations ext4_dir_inode_operations = {
 	.listxattr	= ext4_listxattr,
 	.removexattr	= generic_removexattr,
 #endif
-	.check_acl	= ext4_check_acl,
+	.check_acl_rcu	= ext4_check_acl_rcu,
 	.fiemap         = ext4_fiemap,
 };
 
@@ -2523,5 +2523,5 @@ const struct inode_operations ext4_special_inode_operations = {
 	.listxattr	= ext4_listxattr,
 	.removexattr	= generic_removexattr,
 #endif
-	.check_acl	= ext4_check_acl,
+	.check_acl_rcu	= ext4_check_acl_rcu,
 };
fs/generic_acl.c +20 -0
index 6bc9e3a..3b250e5 100644
@@ -190,6 +190,26 @@ generic_acl_chmod(struct inode *inode)
 }
 
 int
+generic_check_acl_rcu(struct inode *inode, int mask, unsigned int flags)
+{
+	if (flags & IPERM_FLAG_RCU) {
+		if (!negative_cached_acl(inode, ACL_TYPE_ACCESS))
+			return -ECHILD;
+	} else {
+		struct posix_acl *acl;
+
+		acl = get_cached_acl(inode, ACL_TYPE_ACCESS);
+
+		if (acl) {
+			int error = posix_acl_permission(inode, acl, mask);
+			posix_acl_release(acl);
+			return error;
+		}
+	}
+	return -EAGAIN;
+}
+
+int
 generic_check_acl(struct inode *inode, int mask)
 {
 	struct posix_acl *acl = get_cached_acl(inode, ACL_TYPE_ACCESS);
fs/xfs/linux-2.6/xfs_acl.c +7 -1
index b277186..70053f6 100644
@@ -219,7 +219,7 @@ xfs_set_acl(struct inode *inode, int type, struct posix_acl *acl)
 }
 
 int
-xfs_check_acl(struct inode *inode, int mask)
+xfs_check_acl_rcu(struct inode *inode, int mask, unsigned int flags)
 {
 	struct xfs_inode *ip = XFS_I(inode);
 	struct posix_acl *acl;
@@ -234,6 +234,12 @@ xfs_check_acl(struct inode *inode, int mask)
 	if (!XFS_IFORK_Q(ip))
 		return -EAGAIN;
 
+	if (flags & IPERM_FLAG_RCU) {
+		if (!negative_cached_acl(inode, ACL_TYPE_ACCESS))
+			return -ECHILD;
+		return -EAGAIN;
+	}
+
 	acl = xfs_get_acl(inode, ACL_TYPE_ACCESS);
 	if (IS_ERR(acl))
 		return PTR_ERR(acl);
fs/xfs/linux-2.6/xfs_iops.c +4 -4
index 94d5fd6..c8751c7 100644
@@ -643,7 +643,7 @@ xfs_vn_fiemap(
 }
 
 static const struct inode_operations xfs_inode_operations = {
-	.check_acl		= xfs_check_acl,
+	.check_acl_rcu		= xfs_check_acl_rcu,
 	.getattr		= xfs_vn_getattr,
 	.setattr		= xfs_vn_setattr,
 	.setxattr		= generic_setxattr,
@@ -670,7 +670,7 @@ static const struct inode_operations xfs_dir_inode_operations = {
 	.rmdir			= xfs_vn_unlink,
 	.mknod			= xfs_vn_mknod,
 	.rename			= xfs_vn_rename,
-	.check_acl		= xfs_check_acl,
+	.check_acl_rcu		= xfs_check_acl_rcu,
 	.getattr		= xfs_vn_getattr,
 	.setattr		= xfs_vn_setattr,
 	.setxattr		= generic_setxattr,
@@ -695,7 +695,7 @@ static const struct inode_operations xfs_dir_ci_inode_operations = {
 	.rmdir			= xfs_vn_unlink,
 	.mknod			= xfs_vn_mknod,
 	.rename			= xfs_vn_rename,
-	.check_acl		= xfs_check_acl,
+	.check_acl_rcu		= xfs_check_acl_rcu,
 	.getattr		= xfs_vn_getattr,
 	.setattr		= xfs_vn_setattr,
 	.setxattr		= generic_setxattr,
@@ -708,7 +708,7 @@ static const struct inode_operations xfs_symlink_inode_operations = {
 	.readlink		= generic_readlink,
 	.follow_link		= xfs_vn_follow_link,
 	.put_link		= xfs_vn_put_link,
-	.check_acl		= xfs_check_acl,
+	.check_acl_rcu		= xfs_check_acl_rcu,
 	.getattr		= xfs_vn_getattr,
 	.setattr		= xfs_vn_setattr,
 	.setxattr		= generic_setxattr,
fs/xfs/xfs_acl.h +2 -2
index 0135e2a..6560d56 100644
@@ -42,7 +42,7 @@ struct xfs_acl {
 #define SGI_ACL_DEFAULT_SIZE	(sizeof(SGI_ACL_DEFAULT)-1)
 
 #ifdef CONFIG_XFS_POSIX_ACL
-extern int xfs_check_acl(struct inode *inode, int mask);
+extern int xfs_check_acl_rcu(struct inode *inode, int mask, unsigned int flags);
 extern struct posix_acl *xfs_get_acl(struct inode *inode, int type);
 extern int xfs_inherit_acl(struct inode *inode, struct posix_acl *default_acl);
 extern int xfs_acl_chmod(struct inode *inode);
@@ -52,7 +52,7 @@ extern int posix_acl_default_exists(struct inode *inode);
 extern const struct xattr_handler xfs_xattr_acl_access_handler;
 extern const struct xattr_handler xfs_xattr_acl_default_handler;
 #else
-# define xfs_check_acl					NULL
+# define xfs_check_acl_rcu					NULL
 # define xfs_get_acl(inode, type)			NULL
 # define xfs_inherit_acl(inode, default_acl)		0
 # define xfs_acl_chmod(inode)				0
include/linux/generic_acl.h +1 -0
index 574bea4..b57e89c 100644
@@ -11,5 +11,6 @@ extern const struct xattr_handler generic_acl_default_handler;
 int generic_acl_init(struct inode *, struct inode *);
 int generic_acl_chmod(struct inode *);
 int generic_check_acl(struct inode *inode, int mask);
+int generic_check_acl_rcu(struct inode *inode, int mask, unsigned int flags);
 
 #endif /* LINUX_GENERIC_ACL_H */
include/linux/posix_acl.h +19 -0
index 6760816..d68283a 100644
@@ -108,6 +108,25 @@ static inline struct posix_acl *get_cached_acl(struct inode *inode, int type)
 	return acl;
 }
 
+static inline int negative_cached_acl(struct inode *inode, int type)
+{
+	struct posix_acl **p, *acl;
+	switch (type) {
+	case ACL_TYPE_ACCESS:
+		p = &inode->i_acl;
+		break;
+	case ACL_TYPE_DEFAULT:
+		p = &inode->i_default_acl;
+		break;
+	default:
+		BUG();
+	}
+	acl = ACCESS_ONCE(*p);
+	if (acl)
+		return 0;
+	return 1;
+}
+
 static inline void set_cached_acl(struct inode *inode,
 				  int type,
 				  struct posix_acl *acl)
mm/shmem.c +3 -3
index 5ee67c9..8f40d25 100644
@@ -2485,7 +2485,7 @@ static const struct inode_operations shmem_inode_operations = {
 	.getxattr	= generic_getxattr,
 	.listxattr	= generic_listxattr,
 	.removexattr	= generic_removexattr,
-	.check_acl	= generic_check_acl,
+	.check_acl_rcu	= generic_check_acl_rcu,
 #endif
 
 };
@@ -2508,7 +2508,7 @@ static const struct inode_operations shmem_dir_inode_operations = {
 	.getxattr	= generic_getxattr,
 	.listxattr	= generic_listxattr,
 	.removexattr	= generic_removexattr,
-	.check_acl	= generic_check_acl,
+	.check_acl_rcu	= generic_check_acl_rcu,
 #endif
 };
 
@@ -2519,7 +2519,7 @@ static const struct inode_operations shmem_special_inode_operations = {
 	.getxattr	= generic_getxattr,
 	.listxattr	= generic_listxattr,
 	.removexattr	= generic_removexattr,
-	.check_acl	= generic_check_acl,
+	.check_acl_rcu	= generic_check_acl_rcu,
 #endif
 };
 
Signature · 1 line
1.7.1
#74 lore raw

PATCH 42/46 kernel: add bl_list

In reply to Nick Piggin

Introduce a type of hlist that can support the use of the lowest bit in the hlist_head. This will be subsequently used to implement per-bucket bit spinlock for inode and dentry hashes, and may be useful in other cases such as network hashes.

---
 include/linux/list_bl.h    |  141 ++++++++++++++++++++++++++++++++++++++++++++
 include/linux/rculist_bl.h |  128 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 269 insertions(+), 0 deletions(-)
 create mode 100644 include/linux/list_bl.h
 create mode 100644 include/linux/rculist_bl.h
include/linux/list_bl.h new file +141 -0
new file mode 100644
index 0000000..c2034b9
@@ -0,0 +1,141 @@
+#ifndef _LINUX_LIST_BL_H
+#define _LINUX_LIST_BL_H
+
+#include <linux/list.h>
+#include <linux/bit_spinlock.h>
+
+/*
+ * Special version of lists, where head of the list has a bit spinlock
+ * in the lowest bit. This is useful for scalable hash tables without
+ * increasing memory footprint overhead.
+ *
+ * For modification operations, the 0 bit of hlist_bl_head->first
+ * pointer must be set.
+ */
+
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
+#define LIST_BL_LOCKMASK	1UL
+#else
+#define LIST_BL_LOCKMASK	0UL
+#endif
+
+#ifdef CONFIG_DEBUG_LIST
+#define LIST_BL_BUG_ON(x) BUG_ON(x)
+#else
+#define LIST_BL_BUG_ON(x)
+#endif
+
+
+struct hlist_bl_head {
+	struct hlist_bl_node *first;
+};
+
+struct hlist_bl_node {
+	struct hlist_bl_node *next, **pprev;
+};
+#define INIT_HLIST_BL_HEAD(ptr) \
+	((ptr)->first = NULL)
+
+static inline void INIT_HLIST_BL_NODE(struct hlist_bl_node *h)
+{
+	h->next = NULL;
+	h->pprev = NULL;
+}
+
+#define hlist_bl_entry(ptr, type, member) container_of(ptr,type,member)
+
+static inline int hlist_bl_unhashed(const struct hlist_bl_node *h)
+{
+	return !h->pprev;
+}
+
+static inline struct hlist_bl_node *hlist_bl_first(struct hlist_bl_head *h)
+{
+	return (struct hlist_bl_node *)
+		((unsigned long)h->first & ~LIST_BL_LOCKMASK);
+}
+
+static inline void hlist_bl_set_first(struct hlist_bl_head *h,
+					struct hlist_bl_node *n)
+{
+	LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK);
+	LIST_BL_BUG_ON(!bit_spin_is_locked(0, (unsigned long *)&h->first));
+	h->first = (struct hlist_bl_node *)((unsigned long)n | LIST_BL_LOCKMASK);
+}
+
+static inline int hlist_bl_empty(const struct hlist_bl_head *h)
+{
+	return !((unsigned long)h->first & ~LIST_BL_LOCKMASK);
+}
+
+static inline void hlist_bl_add_head(struct hlist_bl_node *n,
+					struct hlist_bl_head *h)
+{
+	struct hlist_bl_node *first = hlist_bl_first(h);
+
+	n->next = first;
+	if (first)
+		first->pprev = &n->next;
+	n->pprev = &h->first;
+	hlist_bl_set_first(h, n);
+}
+
+static inline void __hlist_bl_del(struct hlist_bl_node *n)
+{
+	struct hlist_bl_node *next = n->next;
+	struct hlist_bl_node **pprev = n->pprev;
+
+	LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK);
+
+	/* pprev may be `first`, so be careful not to lose the lock bit */
+	*pprev = (struct hlist_bl_node *)
+			((unsigned long)next |
+			 ((unsigned long)*pprev & LIST_BL_LOCKMASK));
+	if (next)
+		next->pprev = pprev;
+}
+
+static inline void hlist_bl_del(struct hlist_bl_node *n)
+{
+	__hlist_bl_del(n);
+	n->next = LIST_POISON1;
+	n->pprev = LIST_POISON2;
+}
+
+static inline void hlist_bl_del_init(struct hlist_bl_node *n)
+{
+	if (!hlist_bl_unhashed(n)) {
+		__hlist_bl_del(n);
+		INIT_HLIST_BL_NODE(n);
+	}
+}
+
+/**
+ * hlist_bl_for_each_entry	- iterate over list of given type
+ * @tpos:	the type * to use as a loop cursor.
+ * @pos:	the &struct hlist_node to use as a loop cursor.
+ * @head:	the head for your list.
+ * @member:	the name of the hlist_node within the struct.
+ *
+ */
+#define hlist_bl_for_each_entry(tpos, pos, head, member)		\
+	for (pos = hlist_bl_first(head);				\
+	     pos &&							\
+		({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1;}); \
+	     pos = pos->next)
+
+/**
+ * hlist_bl_for_each_entry_safe - iterate over list of given type safe against removal of list entry
+ * @tpos:	the type * to use as a loop cursor.
+ * @pos:	the &struct hlist_node to use as a loop cursor.
+ * @n:		another &struct hlist_node to use as temporary storage
+ * @head:	the head for your list.
+ * @member:	the name of the hlist_node within the struct.
+ */
+#define hlist_bl_for_each_entry_safe(tpos, pos, n, head, member)	 \
+	for (pos = hlist_bl_first(head);				 \
+	     pos && ({ n = pos->next; 1; }) && 				 \
+		({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1;}); \
+	     pos = n)
+
+#endif
include/linux/rculist_bl.h new file +128 -0

128 lines not on this page · raw message

Signature · 1 line
1.7.1
#75 lore raw

PATCH 43/46 bit_spinlock: add required includes

In reply to Nick Piggin
---
 include/linux/bit_spinlock.h |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)
include/linux/bit_spinlock.h +4 -0
index 7113a32..e612575 100644
@@ -1,6 +1,10 @@
 #ifndef __LINUX_BIT_SPINLOCK_H
 #define __LINUX_BIT_SPINLOCK_H
 
+#include <linux/kernel.h>
+#include <linux/preempt.h>
+#include <asm/atomic.h>
+
 /*
  *  bit-based spin_lock()
  *
Signature · 1 line
1.7.1
#76 lore raw

PATCH 44/46 fs: dcache per-bucket dcache hash locking

In reply to Nick Piggin

We can turn the dcache hash locking from a global dcache_hash_lock into per-bucket locking.

---
 fs/dcache.c            |  131 ++++++++++++++++++++++++++++++++---------------
 fs/super.c             |    3 +-
 include/linux/dcache.h |   23 ++-------
 include/linux/fs.h     |    3 +-
 4 files changed, 97 insertions(+), 63 deletions(-)
fs/dcache.c +89 -42
index 67a08d4..5e19940 100644
@@ -33,13 +33,15 @@
 #include <linux/bootmem.h>
 #include <linux/fs_struct.h>
 #include <linux/hardirq.h>
+#include <linux/bit_spinlock.h>
+#include <linux/rculist_bl.h>
 #include "internal.h"
 
 /*
  * Usage:
  * dcache_inode_lock protects:
  *   - i_dentry, d_alias, d_inode
- * dcache_hash_lock protects:
+ * dcache_hash_bucket lock protects:
  *   - the dcache hash table
  * dcache_lru_lock protects:
  *   - the dcache lru lists and counters
@@ -57,7 +59,7 @@
  * dcache_inode_lock
  *   dentry->d_lock
  *     dcache_lru_lock
- *     dcache_hash_lock
+ *     dcache_hash_bucket lock
  *
  * If there is an ancestor relationship:
  * dentry->d_parent->...->d_parent->d_lock
@@ -74,13 +76,11 @@ int sysctl_vfs_cache_pressure __read_mostly = 100;
 EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
 
 __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_inode_lock);
-__cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_hash_lock);
 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lru_lock);
 __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
 
 EXPORT_SYMBOL(rename_lock);
 EXPORT_SYMBOL(dcache_inode_lock);
-EXPORT_SYMBOL(dcache_hash_lock);
 
 static struct kmem_cache *dentry_cache __read_mostly;
 
@@ -97,13 +97,35 @@ static struct kmem_cache *dentry_cache __read_mostly;
 
 static unsigned int d_hash_mask __read_mostly;
 static unsigned int d_hash_shift __read_mostly;
-static struct hlist_head *dentry_hashtable __read_mostly;
+
+struct dcache_hash_bucket {
+	struct hlist_bl_head head;
+};
+static struct dcache_hash_bucket *dentry_hashtable __read_mostly;
 
 /* Statistics gathering. */
 struct dentry_stat_t dentry_stat = {
 	.age_limit = 45,
 };
 
+static inline struct dcache_hash_bucket *d_hash(struct dentry *parent,
+					unsigned long hash)
+{
+	hash += ((unsigned long) parent ^ GOLDEN_RATIO_PRIME) / L1_CACHE_BYTES;
+	hash = hash ^ ((hash ^ GOLDEN_RATIO_PRIME) >> D_HASHBITS);
+	return dentry_hashtable + (hash & D_HASHMASK);
+}
+
+static inline void spin_lock_bucket(struct dcache_hash_bucket *b)
+{
+	bit_spin_lock(0, (unsigned long *)b);
+}
+
+static inline void spin_unlock_bucket(struct dcache_hash_bucket *b)
+{
+	__bit_spin_unlock(0, (unsigned long *)b);
+}
+
 static struct percpu_counter nr_dentry __cacheline_aligned_in_smp;
 static struct percpu_counter nr_dentry_unused __cacheline_aligned_in_smp;
 
@@ -138,7 +160,7 @@ static void d_free(struct dentry *dentry)
 		dentry->d_op->d_release(dentry);
 
 	/* if dentry was never inserted into hash, immediate free is OK */
-	if (hlist_unhashed(&dentry->d_hash))
+	if (hlist_bl_unhashed(&dentry->d_hash))
 		__d_free(&dentry->d_u.d_rcu);
 	else
 		call_rcu(&dentry->d_u.d_rcu, __d_free);
@@ -278,6 +300,39 @@ relock:
 	return d_kill(dentry, parent);
 }
 
+void __d_drop(struct dentry *dentry)
+{
+	if (!(dentry->d_flags & DCACHE_UNHASHED)) {
+		if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED)) {
+			bit_spin_lock(0, (unsigned long *)&dentry->d_sb->s_anon);
+			dentry->d_flags |= DCACHE_UNHASHED;
+			hlist_bl_del_init(&dentry->d_hash);
+			__bit_spin_unlock(0, (unsigned long *)&dentry->d_sb->s_anon);
+		} else {
+			struct dcache_hash_bucket *b;
+			b = d_hash(dentry->d_parent, dentry->d_name.hash);
+			spin_lock_bucket(b);
+			/*
+			 * We may not actually need to put DCACHE_UNHASHED
+			 * manipulations under the hash lock, but follow
+			 * the principle of least surprise.
+			 */
+			dentry->d_flags |= DCACHE_UNHASHED;
+			hlist_bl_del_rcu(&dentry->d_hash);
+			spin_unlock_bucket(b);
+		}
+	}
+}
+EXPORT_SYMBOL(__d_drop);
+
+void d_drop(struct dentry *dentry)
+{
+	spin_lock(&dentry->d_lock);
+ 	__d_drop(dentry);
+	spin_unlock(&dentry->d_lock);
+}
+EXPORT_SYMBOL(d_drop);
+
 /* 
  * This is dput
  *
@@ -891,8 +946,8 @@ void shrink_dcache_for_umount(struct super_block *sb)
 	spin_unlock(&dentry->d_lock);
 	shrink_dcache_for_umount_subtree(dentry);
 
-	while (!hlist_empty(&sb->s_anon)) {
-		dentry = hlist_entry(sb->s_anon.first, struct dentry, d_hash);
+	while (!hlist_bl_empty(&sb->s_anon)) {
+		dentry = hlist_bl_entry(hlist_bl_first(&sb->s_anon), struct dentry, d_hash);
 		shrink_dcache_for_umount_subtree(dentry);
 	}
 }
@@ -1196,7 +1251,7 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
 	dentry->d_sb = NULL;
 	dentry->d_op = NULL;
 	dentry->d_fsdata = NULL;
-	INIT_HLIST_NODE(&dentry->d_hash);
+	INIT_HLIST_BL_NODE(&dentry->d_hash);
 	INIT_LIST_HEAD(&dentry->d_lru);
 	INIT_LIST_HEAD(&dentry->d_subdirs);
 	INIT_LIST_HEAD(&dentry->d_alias);
@@ -1387,14 +1442,6 @@ struct dentry * d_alloc_root(struct inode * root_inode)
 }
 EXPORT_SYMBOL(d_alloc_root);
 
-static inline struct hlist_head *d_hash(struct dentry *parent,
-					unsigned long hash)
-{
-	hash += ((unsigned long) parent ^ GOLDEN_RATIO_PRIME) / L1_CACHE_BYTES;
-	hash = hash ^ ((hash ^ GOLDEN_RATIO_PRIME) >> D_HASHBITS);
-	return dentry_hashtable + (hash & D_HASHMASK);
-}
-
 /**
  * d_obtain_alias - find or allocate a dentry for a given inode
  * @inode: inode to allocate the dentry for
@@ -1449,11 +1496,11 @@ struct dentry *d_obtain_alias(struct inode *inode)
 	tmp->d_sb = inode->i_sb;
 	tmp->d_inode = inode;
 	tmp->d_flags |= DCACHE_DISCONNECTED;
-	tmp->d_flags &= ~DCACHE_UNHASHED;
 	list_add(&tmp->d_alias, &inode->i_dentry);
-	spin_lock(&dcache_hash_lock);
-	hlist_add_head(&tmp->d_hash, &inode->i_sb->s_anon);
-	spin_unlock(&dcache_hash_lock);
+	bit_spin_lock(0, (unsigned long *)&tmp->d_sb->s_anon);
+	tmp->d_flags &= ~DCACHE_UNHASHED;
+	hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon);
+	__bit_spin_unlock(0, (unsigned long *)&tmp->d_sb->s_anon);
 	spin_unlock(&tmp->d_lock);
 	spin_unlock(&dcache_inode_lock);
 
@@ -1617,8 +1664,8 @@ struct dentry *__d_lookup_rcu(struct dentry *parent, struct qstr *name,
 	unsigned int len = name->len;
 	unsigned int hash = name->hash;
 	const unsigned char *str = name->name;
-	struct hlist_head *head = d_hash(parent,hash);
-	struct hlist_node *node;
+	struct dcache_hash_bucket *b = d_hash(parent, hash);
+	struct hlist_bl_node *node;
 	struct dentry *dentry;
 
  	/*
@@ -1641,7 +1688,7 @@ struct dentry *__d_lookup_rcu(struct dentry *parent, struct qstr *name,
 	 *
 	 * See Documentation/vfs/dcache-locking.txt for more details.
 	 */
-	hlist_for_each_entry_rcu(dentry, node, head, d_hash) {
+	hlist_bl_for_each_entry_rcu(dentry, node, &b->head, d_hash) {
 		struct inode *i;
 		const char *tname;
 		int tlen;
@@ -1754,8 +1801,8 @@ struct dentry *__d_lookup(struct dentry *parent, struct qstr *name)
 	unsigned int len = name->len;
 	unsigned int hash = name->hash;
 	const unsigned char *str = name->name;
-	struct hlist_head *head = d_hash(parent,hash);
-	struct hlist_node *node;
+	struct dcache_hash_bucket *b = d_hash(parent, hash);
+	struct hlist_bl_node *node;
 	struct dentry *found = NULL;
 	struct dentry *dentry;
 
@@ -1781,7 +1828,7 @@ struct dentry *__d_lookup(struct dentry *parent, struct qstr *name)
 	 */
 	rcu_read_lock();
 	
-	hlist_for_each_entry_rcu(dentry, node, head, d_hash) {
+	hlist_bl_for_each_entry_rcu(dentry, node, &b->head, d_hash) {
 		const char *tname;
 		int tlen;
 
@@ -1932,11 +1979,12 @@ again:
 }
 EXPORT_SYMBOL(d_delete);
 
-static void __d_rehash(struct dentry * entry, struct hlist_head *list)
+static void __d_rehash(struct dentry * entry, struct dcache_hash_bucket *b)
 {
-
+	spin_lock_bucket(b);
  	entry->d_flags &= ~DCACHE_UNHASHED;
- 	hlist_add_head_rcu(&entry->d_hash, list);
+ 	hlist_bl_add_head_rcu(&entry->d_hash, &b->head);
+	spin_unlock_bucket(b);
 }
 
 static void _d_rehash(struct dentry * entry)
@@ -1954,9 +2002,7 @@ static void _d_rehash(struct dentry * entry)
 void d_rehash(struct dentry * entry)
 {
 	spin_lock(&entry->d_lock);
-	spin_lock(&dcache_hash_lock);
 	_d_rehash(entry);
-	spin_unlock(&dcache_hash_lock);
 	spin_unlock(&entry->d_lock);
 }
 EXPORT_SYMBOL(d_rehash);
@@ -2035,6 +2081,7 @@ static void switch_names(struct dentry *dentry, struct dentry *target)
  */
 void d_move(struct dentry * dentry, struct dentry * target)
 {
+	struct dcache_hash_bucket *b;
 	if (!dentry->d_inode)
 		printk(KERN_WARNING "VFS: moving negative dcache entry\n");
 
@@ -2065,11 +2112,13 @@ void d_move(struct dentry * dentry, struct dentry * target)
 	}
 
 	/* Move the dentry to the target hash queue, if on different bucket */
-	spin_lock(&dcache_hash_lock);
-	if (!d_unhashed(dentry))
-		hlist_del_rcu(&dentry->d_hash);
+	if (!d_unhashed(dentry)) {
+		b = d_hash(dentry->d_parent, dentry->d_name.hash);
+		spin_lock_bucket(b);
+		hlist_bl_del_rcu(&dentry->d_hash);
+		spin_unlock_bucket(b);
+	}
 	__d_rehash(dentry, d_hash(target->d_parent, target->d_name.hash));
-	spin_unlock(&dcache_hash_lock);
 
 	/* Unhash the target: dput() will then get rid of it */
 	__d_drop(target);
@@ -2280,9 +2329,7 @@ struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode)
 
 	spin_lock(&actual->d_lock);
 found:
-	spin_lock(&dcache_hash_lock);
 	_d_rehash(actual);
-	spin_unlock(&dcache_hash_lock);
 	spin_unlock(&actual->d_lock);
 	spin_unlock(&dcache_inode_lock);
 out_nolock:
@@ -2864,7 +2911,7 @@ static void __init dcache_init_early(void)
 
 	dentry_hashtable =
 		alloc_large_system_hash("Dentry cache",
-					sizeof(struct hlist_head),
+					sizeof(struct dcache_hash_bucket),
 					dhash_entries,
 					13,
 					HASH_EARLY,
@@ -2873,7 +2920,7 @@ static void __init dcache_init_early(void)
 					0);
 
 	for (loop = 0; loop < (1 << d_hash_shift); loop++)
-		INIT_HLIST_HEAD(&dentry_hashtable[loop]);
+		INIT_HLIST_BL_HEAD(&dentry_hashtable[loop].head);
 }
 
 static void __init dcache_init(void)
@@ -2899,7 +2946,7 @@ static void __init dcache_init(void)
 
 	dentry_hashtable =
 		alloc_large_system_hash("Dentry cache",
-					sizeof(struct hlist_head),
+					sizeof(struct dcache_hash_bucket),
 					dhash_entries,
 					13,
 					0,
@@ -2908,7 +2955,7 @@ static void __init dcache_init(void)
 					0);
 
 	for (loop = 0; loop < (1 << d_hash_shift); loop++)
-		INIT_HLIST_HEAD(&dentry_hashtable[loop]);
+		INIT_HLIST_BL_HEAD(&dentry_hashtable[loop].head);
 }
 
 /* SLAB cache for __getname() consumers */
fs/super.c +2 -1
index ca69615..968ba01 100644
@@ -30,6 +30,7 @@
 #include <linux/idr.h>
 #include <linux/mutex.h>
 #include <linux/backing-dev.h>
+#include <linux/rculist_bl.h>
 #include "internal.h"
 
 
@@ -71,7 +72,7 @@ static struct super_block *alloc_super(struct file_system_type *type)
 		INIT_LIST_HEAD(&s->s_files);
 #endif
 		INIT_LIST_HEAD(&s->s_instances);
-		INIT_HLIST_HEAD(&s->s_anon);
+		INIT_HLIST_BL_HEAD(&s->s_anon);
 		INIT_LIST_HEAD(&s->s_inodes);
 		INIT_LIST_HEAD(&s->s_dentry_lru);
 		init_rwsem(&s->s_umount);
include/linux/dcache.h +4 -19
index 72f5f32..97c2d78 100644
@@ -4,6 +4,7 @@
 #include <asm/atomic.h>
 #include <linux/list.h>
 #include <linux/rculist.h>
+#include <linux/rculist_bl.h>
 #include <linux/spinlock.h>
 #include <linux/seqlock.h>
 #include <linux/cache.h>
@@ -91,7 +92,7 @@ struct dentry {
 	/* RCU lookup touched fields */
 	unsigned int d_flags;		/* protected by d_lock */
 	seqcount_t d_seq;		/* per dentry seqlock */
-	struct hlist_node d_hash;	/* lookup hash list */
+	struct hlist_bl_node d_hash;	/* lookup hash list */
 	struct dentry *d_parent;	/* parent directory */
 	struct qstr d_name;
 	struct inode *d_inode;		/* Where the name belongs to - NULL is
@@ -193,7 +194,6 @@ struct dentry_operations {
 	(DCACHE_OP_REVALIDATE|DCACHE_OP_REVALIDATE_RCU)
 
 extern spinlock_t dcache_inode_lock;
-extern spinlock_t dcache_hash_lock;
 extern seqlock_t rename_lock;
 
 /**
@@ -211,23 +211,8 @@ extern seqlock_t rename_lock;
  *
  * __d_drop requires dentry->d_lock.
  */
-
-static inline void __d_drop(struct dentry *dentry)
-{
-	if (!(dentry->d_flags & DCACHE_UNHASHED)) {
-		dentry->d_flags |= DCACHE_UNHASHED;
-		spin_lock(&dcache_hash_lock);
-		hlist_del_rcu(&dentry->d_hash);
-		spin_unlock(&dcache_hash_lock);
-	}
-}
-
-static inline void d_drop(struct dentry *dentry)
-{
-	spin_lock(&dentry->d_lock);
- 	__d_drop(dentry);
-	spin_unlock(&dentry->d_lock);
-}
+void d_drop(struct dentry *dentry);
+void __d_drop(struct dentry *dentry);
 
 static inline int dname_external(struct dentry *dentry)
 {
include/linux/fs.h +2 -1
index 490eedd..315d0e9 100644
@@ -392,6 +392,7 @@ struct inodes_stat_t {
 #include <linux/capability.h>
 #include <linux/semaphore.h>
 #include <linux/fiemap.h>
+#include <linux/rculist_bl.h>
 
 #include <asm/atomic.h>
 #include <asm/byteorder.h>
@@ -1376,7 +1377,7 @@ struct super_block {
 	const struct xattr_handler **s_xattr;
 
 	struct list_head	s_inodes;	/* all inodes */
-	struct hlist_head	s_anon;		/* anonymous dentries for (nfs) exporting */
+	struct hlist_bl_head	s_anon;		/* anonymous dentries for (nfs) exporting */
 #ifdef CONFIG_SMP
 	struct list_head __percpu *s_files;
 #else
Signature · 1 line
1.7.1
#77 lore raw

PATCH 45/46 fs: dcache per-inode inode alias locking

In reply to Nick Piggin

dcache_inode_lock can be replaced with per-inode locking. Use existing inode->i_lock for this. This is slightly non-trivial because we sometimes need to find the inode from the dentry, which requires d_inode to be stabilised (either with refcount or d_lock).

---
 fs/9p/vfs_inode.c      |    4 +-
 fs/affs/amigaffs.c     |    4 +-
 fs/dcache.c            |   84 +++++++++++++++++++++++++----------------------
 fs/exportfs/expfs.c    |   12 ++++---
 fs/nfs/getroot.c       |    4 +-
 fs/notify/fsnotify.c   |    4 +-
 fs/ocfs2/dcache.c      |    4 +-
 include/linux/dcache.h |    1 -
 8 files changed, 62 insertions(+), 55 deletions(-)
fs/9p/vfs_inode.c +2 -2
index df8bbb3..5978298 100644
@@ -277,11 +277,11 @@ static struct dentry *v9fs_dentry_from_dir_inode(struct inode *inode)
 {
 	struct dentry *dentry;
 
-	spin_lock(&dcache_inode_lock);
+	spin_lock(&inode->i_lock);
 	/* Directory should have only one entry. */
 	BUG_ON(S_ISDIR(inode->i_mode) && !list_is_singular(&inode->i_dentry));
 	dentry = list_entry(inode->i_dentry.next, struct dentry, d_alias);
-	spin_unlock(&dcache_inode_lock);
+	spin_unlock(&inode->i_lock);
 	return dentry;
 }
 
fs/affs/amigaffs.c +2 -2
index 600101a..3a4557e 100644
@@ -128,7 +128,7 @@ affs_fix_dcache(struct dentry *dentry, u32 entry_ino)
 	void *data = dentry->d_fsdata;
 	struct list_head *head, *next;
 
-	spin_lock(&dcache_inode_lock);
+	spin_lock(&inode->i_lock);
 	head = &inode->i_dentry;
 	next = head->next;
 	while (next != head) {
@@ -139,7 +139,7 @@ affs_fix_dcache(struct dentry *dentry, u32 entry_ino)
 		}
 		next = next->next;
 	}
-	spin_unlock(&dcache_inode_lock);
+	spin_unlock(&inode->i_lock);
 }
 
 
fs/dcache.c +45 -39

271 lines not on this page · raw message

fs/exportfs/expfs.c +7 -5
index f06a940..4b68257 100644
@@ -43,24 +43,26 @@ find_acceptable_alias(struct dentry *result,
 		void *context)
 {
 	struct dentry *dentry, *toput = NULL;
+	struct inode *inode;
 
 	if (acceptable(context, result))
 		return result;
 
-	spin_lock(&dcache_inode_lock);
-	list_for_each_entry(dentry, &result->d_inode->i_dentry, d_alias) {
+	inode = result->d_inode;
+	spin_lock(&inode->i_lock);
+	list_for_each_entry(dentry, &inode->i_dentry, d_alias) {
 		dget(dentry);
-		spin_unlock(&dcache_inode_lock);
+		spin_unlock(&inode->i_lock);
 		if (toput)
 			dput(toput);
 		if (dentry != result && acceptable(context, dentry)) {
 			dput(result);
 			return dentry;
 		}
-		spin_lock(&dcache_inode_lock);
+		spin_lock(&inode->i_lock);
 		toput = dentry;
 	}
-	spin_unlock(&dcache_inode_lock);
+	spin_unlock(&inode->i_lock);
 
 	if (toput)
 		dput(toput);
fs/nfs/getroot.c +2 -2
index c3a5a11..5596c6a 100644
@@ -63,11 +63,11 @@ static int nfs_superblock_set_dummy_root(struct super_block *sb, struct inode *i
 		 * This again causes shrink_dcache_for_umount_subtree() to
 		 * Oops, since the test for IS_ROOT() will fail.
 		 */
-		spin_lock(&dcache_inode_lock);
+		spin_lock(&sb->s_root->d_inode->i_lock);
 		spin_lock(&sb->s_root->d_lock);
 		list_del_init(&sb->s_root->d_alias);
 		spin_unlock(&sb->s_root->d_lock);
-		spin_unlock(&dcache_inode_lock);
+		spin_unlock(&sb->s_root->d_inode->i_lock);
 	}
 	return 0;
 }
fs/notify/fsnotify.c +2 -2
index 9be6ec1..79b47cb 100644
@@ -59,7 +59,7 @@ void __fsnotify_update_child_dentry_flags(struct inode *inode)
 	/* determine if the children should tell inode about their events */
 	watched = fsnotify_inode_watches_children(inode);
 
-	spin_lock(&dcache_inode_lock);
+	spin_lock(&inode->i_lock);
 	/* run all of the dentries associated with this inode.  Since this is a
 	 * directory, there damn well better only be one item on this list */
 	list_for_each_entry(alias, &inode->i_dentry, d_alias) {
@@ -82,7 +82,7 @@ void __fsnotify_update_child_dentry_flags(struct inode *inode)
 		}
 		spin_unlock(&alias->d_lock);
 	}
-	spin_unlock(&dcache_inode_lock);
+	spin_unlock(&inode->i_lock);
 }
 
 /* Notify this dentry's parent about a child's events. */
fs/ocfs2/dcache.c +2 -2
index 8b26b54..b4d9421 100644
@@ -169,7 +169,7 @@ struct dentry *ocfs2_find_local_alias(struct inode *inode,
 	struct list_head *p;
 	struct dentry *dentry = NULL;
 
-	spin_lock(&dcache_inode_lock);
+	spin_lock(&inode->i_lock);
 	list_for_each(p, &inode->i_dentry) {
 		dentry = list_entry(p, struct dentry, d_alias);
 
@@ -187,7 +187,7 @@ struct dentry *ocfs2_find_local_alias(struct inode *inode,
 		dentry = NULL;
 	}
 
-	spin_unlock(&dcache_inode_lock);
+	spin_unlock(&inode->i_lock);
 
 	return dentry;
 }
include/linux/dcache.h +0 -1
index 97c2d78..8ff803b 100644
@@ -193,7 +193,6 @@ struct dentry_operations {
 #define DCACHE_OP_REVALIDATE_EITHER	\
 	(DCACHE_OP_REVALIDATE|DCACHE_OP_REVALIDATE_RCU)
 
-extern spinlock_t dcache_inode_lock;
 extern seqlock_t rename_lock;
 
 /**
Signature · 1 line
1.7.1
#78 lore raw

PATCH 46/46 fs: improve scalability of pseudo filesystems

In reply to Nick Piggin

Regardless of how much we possibly try to scale dcache, there is likely always going to be some fundamental contention when adding or removing children under the same parent. Pseudo filesystems do not seem need to have connected dentries because by definition they are disconnected.

---
 fs/anon_inodes.c       |    2 +-
 fs/dcache.c            |   12 ++++++++++++
 fs/pipe.c              |    2 +-
 include/linux/dcache.h |    1 +
 net/socket.c           |    2 +-
 5 files changed, 16 insertions(+), 3 deletions(-)
fs/anon_inodes.c +1 -1
index aca8806..9d92b33 100644
@@ -102,7 +102,7 @@ struct file *anon_inode_getfile(const char *name,
 	this.name = name;
 	this.len = strlen(name);
 	this.hash = 0;
-	path.dentry = d_alloc(anon_inode_mnt->mnt_sb->s_root, &this);
+	path.dentry = d_alloc_pseudo(anon_inode_mnt->mnt_sb, &this);
 	if (!path.dentry)
 		goto err_module;
 
fs/dcache.c +12 -0
index 809ec46..59f25b7 100644
@@ -1276,6 +1276,18 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
 }
 EXPORT_SYMBOL(d_alloc);
 
+struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
+{
+	struct dentry *dentry = d_alloc(NULL, name);
+	if (dentry) {
+		dentry->d_sb = sb;
+		dentry->d_parent = dentry;
+		dentry->d_flags |= DCACHE_DISCONNECTED;
+	}
+	return dentry;
+}
+EXPORT_SYMBOL(d_alloc_pseudo);
+
 struct dentry *d_alloc_name(struct dentry *parent, const char *name)
 {
 	struct qstr q;
fs/pipe.c +1 -1
index e964d09..ebe9143 100644
@@ -999,7 +999,7 @@ struct file *create_write_pipe(int flags)
 		goto err;
 
 	err = -ENOMEM;
-	path.dentry = d_alloc(pipe_mnt->mnt_sb->s_root, &name);
+	path.dentry = d_alloc_pseudo(pipe_mnt->mnt_sb, &name);
 	if (!path.dentry)
 		goto err_inode;
 	path.mnt = mntget(pipe_mnt);
include/linux/dcache.h +1 -0
index 8ff803b..0cf2cb9 100644
@@ -229,6 +229,7 @@ extern void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op
 
 /* allocate/de-allocate */
 extern struct dentry * d_alloc(struct dentry *, const struct qstr *);
+extern struct dentry * d_alloc_pseudo(struct super_block *, const struct qstr *);
 extern struct dentry * d_splice_alias(struct inode *, struct dentry *);
 extern struct dentry * d_add_ci(struct dentry *, struct inode *, struct qstr *);
 extern struct dentry * d_obtain_alias(struct inode *);
net/socket.c +1 -1
index 2d8b4c8..17d9aae 100644
@@ -361,7 +361,7 @@ static int sock_alloc_file(struct socket *sock, struct file **f, int flags)
 	if (unlikely(fd < 0))
 		return fd;
 
-	path.dentry = d_alloc(sock_mnt->mnt_sb->s_root, &name);
+	path.dentry = d_alloc_pseudo(sock_mnt->mnt_sb, &name);
 	if (unlikely(!path.dentry)) {
 		put_unused_fd(fd);
 		return -ENOMEM;
Signature · 1 line
1.7.1
#79 lore raw
In reply to Nick Piggin
Quoted 2 lines · Nick Piggin

git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.git vfs-scale-working

I get:
CC [M] fs/cifs/inode.o
fs/cifs/inode.c: In function ‘inode_has_hashed_dentries’:
fs/cifs/inode.c:807: error: ‘dcache_inode_lock’ undeclared (first use in this function)
fs/cifs/inode.c:807: error: (Each undeclared identifier is reported only once fs/cifs/inode.c:807: error: for each function it appears in.) make[3]: *** [fs/cifs/inode.o] Error 1
make[2]: *** [fs/cifs] Error 2
make[1]: *** [fs] Error 2

I used the latest mainline.
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#80 lore raw
Quoted 15 lines · Anca Emanuel
Quoted 2 lines · Nick Piggin

git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.git vfs-scale-working

I get:
CC [M] fs/cifs/inode.o
fs/cifs/inode.c: In function ‘inode_has_hashed_dentries’:
fs/cifs/inode.c:807: error: ‘dcache_inode_lock’ undeclared (first use in this function)
fs/cifs/inode.c:807: error: (Each undeclared identifier is reported only once fs/cifs/inode.c:807: error: for each function it appears in.) make[3]: *** [fs/cifs/inode.o] Error 1
make[2]: *** [fs/cifs] Error 2
make[1]: *** [fs] Error 2

Quoted 1 line

I used the latest mainline.

Sorry, missed a conversion, it just needs to be changed to inode->i_lock. Pushed the fix to git.

Thanks,
Nick

--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#81 lore raw
Quoted 16 lines · Nick Piggin
Quoted 15 lines · Anca Emanuel
Quoted 2 lines · Nick Piggin

git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.git vfs-scale-working

I get:
  CC [M]  fs/cifs/inode.o
fs/cifs/inode.c: In function ‘inode_has_hashed_dentries’:
fs/cifs/inode.c:807: error: ‘dcache_inode_lock’ undeclared (first use in this function)
fs/cifs/inode.c:807: error: (Each undeclared identifier is reported only once fs/cifs/inode.c:807: error: for each function it appears in.) make[3]: *** [fs/cifs/inode.o] Error 1
make[2]: *** [fs/cifs] Error 2
make[1]: *** [fs] Error 2

Quoted 5 lines
Quoted 1 line

I used the latest mainline.

Sorry, missed a conversion, it just needs to be changed to inode->i_lock. Pushed the fix to git.

I attached a patch to my posting in [1] but it was somehow "eaten" (here in my mbox the patch is definitely attached). Patchwork is also not listing my patch.
Anyway, it's fixed - that's good.

I have to check why I get a Call trace with systemd-v15 but not with sysvinit package here on Debian.

- Sedat -

[1] http://lkml.org/lkml/2010/11/27/145

fs/cifs/inode.c +3 -3
Index: linux-2.6/fs/cifs/inode.c
===================================================================
@@ -804,14 +804,14 @@ inode_has_hashed_dentries(struct inode *
 {
 	struct dentry *dentry;
 
-	spin_lock(&dcache_inode_lock);
+	spin_lock(&inode->i_lock);
 	list_for_each_entry(dentry, &inode->i_dentry, d_alias) {
 		if (!d_unhashed(dentry) || IS_ROOT(dentry)) {
-			spin_unlock(&dcache_inode_lock);
+			spin_unlock(&inode->i_lock);
 			return true;
 		}
 	}
-	spin_unlock(&dcache_inode_lock);
+	spin_unlock(&inode->i_lock);
 	return false;
 }
 
maintainer
#82 lore raw
In reply to Nick Piggin

From: Nick Piggin <[email protected]>
Date: Sat, 27 Nov 2010 21:15:58 +1100

Quoted 1 line

git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.git vfs-scale-working

Just want to say that I've been running this code for the past few days on my 128 cpu box and it seems quite sturdy.

If there are any kinds of vfs benchmarks you want me to run on this machine both with and without the scaling changes, just let me know.

#83 lore raw
Quoted 10 lines · David Miller

From: Nick Piggin <[email protected]>
Date: Sat, 27 Nov 2010 21:15:58 +1100

Quoted 1 line

git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.git vfs-scale-working

Just want to say that I've been running this code for the past few days on my 128 cpu box and it seems quite sturdy.

If there are any kinds of vfs benchmarks you want me to run on this machine both with and without the scaling changes, just let me know.

Hi Dave, great, thanks!

I am mostly interested in single-thread performance on different ISAs and different microarchitectures.

Microbenchmarks are easy, I attached a couple of quick ones I use in an offline mail.

Single-threaded (preloadindex=false), cached and uncached, git diff on an unmodified tree is slightly more real world.

And then anything else you can think of.

Thanks,
Nick

#84 lore raw
In reply to Nick Piggin
Quoted 11 lines · Nick Piggin

git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.git vfs-scale-working

Here is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:

* preparation patches
* introduce new locks to take over dcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking steps

Just got this set of traces doing an 8-way parallel remove of 50 million inodes at about 40M inodes unlinked:

[ 5954.061633] BUG: sleeping function called from invalid context at arch/x86/mm/fault.c:1081
[ 5954.062466] in_atomic(): 0, irqs_disabled(): 1, pid: 2927, name: rm
[ 5954.063122] 3 locks held by rm/2927:
[ 5954.063476]  #0:  (&sb->s_type->i_mutex_key#12/1){+.+.+.}, at: [<ffffffff8116f5e1>] do_rmdir+0x81/0x130
[ 5954.064014]  #1:  (&sb->s_type->i_mutex_key#12){+.+.+.}, at: [<ffffffff8116d3a8>] vfs_rmdir+0x58/0xe0
[ 5954.064014]  #2:  (rcu_read_lock){.+.+..}, at: [<ffffffff811779c0>] shrink_dentry_list+0x0/0x430
[ 5954.064014] irq event stamp: 1484376719
[ 5954.064014] hardirqs last  enabled at (1484376719): [<ffffffff810ebf07>] __call_rcu+0xd7/0x1a0
[ 5954.064014] hardirqs last disabled at (1484376718): [<ffffffff810ebe7a>] __call_rcu+0x4a/0x1a0
[ 5954.064014] softirqs last  enabled at (1484376586): [<ffffffff8108b911>] __do_softirq+0x161/0x270
[ 5954.064014] softirqs last disabled at (1484376581): [<ffffffff8103af1c>] call_softirq+0x1c/0x50
[ 5954.064014] Pid: 2927, comm: rm Not tainted 2.6.37-rc4-dgc+ #794
[ 5954.064014] Call Trace:
[ 5954.064014]  [<ffffffff810b95b0>] ? print_irqtrace_events+0xd0/0xe0
[ 5954.064014]  [<ffffffff81076455>] __might_sleep+0xf5/0x130
[ 5954.064014]  [<ffffffff81b1e603>] do_page_fault+0x103/0x4f0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff81b19b28>] ? trace_hardirqs_off_thunk+0x3a/0x3c
[ 5954.064014]  [<ffffffff810b8e20>] ? usage_match+0x0/0x20
[ 5954.064014]  [<ffffffff81b1af25>] page_fault+0x25/0x30
[ 5954.064014]  [<ffffffff810b8e20>] ? usage_match+0x0/0x20
[ 5954.064014]  [<ffffffff810ba1c8>] ? __bfs+0xc8/0x260
[ 5954.064014]  [<ffffffff810ba123>] ? __bfs+0x23/0x260
[ 5954.064014]  [<ffffffff810ba4d2>] find_usage_backwards+0x42/0x80
[ 5954.064014]  [<ffffffff810bcec4>] check_usage_backwards+0x64/0xf0
[ 5954.064014]  [<ffffffff8104796f>] ? save_stack_trace+0x2f/0x50
[ 5954.064014]  [<ffffffff810bce60>] ? check_usage_backwards+0x0/0xf0
[ 5954.064014]  [<ffffffff810bd9a9>] mark_lock+0x1a9/0x440
[ 5954.064014]  [<ffffffff810be989>] __lock_acquire+0x5a9/0x14b0
[ 5954.064014]  [<ffffffff810be716>] ? __lock_acquire+0x336/0x14b0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff810bf944>] lock_acquire+0xb4/0x140
[ 5954.064014]  [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.064014]  [<ffffffff81b19d86>] _raw_spin_lock+0x36/0x70
[ 5954.064014]  [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.064014]  [<ffffffff81177a1c>] shrink_dentry_list+0x5c/0x430
[ 5954.064014]  [<ffffffff811779c0>] ? shrink_dentry_list+0x0/0x430
[ 5954.064014]  [<ffffffff816b9c7e>] ? do_raw_spin_unlock+0x5e/0xb0
[ 5954.064014]  [<ffffffff81177f2d>] __shrink_dcache_sb+0x13d/0x1c0
[ 5954.064014]  [<ffffffff811784bf>] shrink_dcache_parent+0x32f/0x390
[ 5954.064014]  [<ffffffff8116d31d>] dentry_unhash+0x3d/0x70
[ 5954.064014]  [<ffffffff8116d3b0>] vfs_rmdir+0x60/0xe0
[ 5954.064014]  [<ffffffff8116f673>] do_rmdir+0x113/0x130
[ 5954.064014]  [<ffffffff8103a03a>] ? sysret_check+0x2e/0x69
[ 5954.064014]  [<ffffffff81b19ae9>] ? trace_hardirqs_on_thunk+0x3a/0x3f
[ 5954.064014]  [<ffffffff8116f6c5>] sys_unlinkat+0x35/0x40
[ 5954.064014]  [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
[ 5954.092916] BUG: unable to handle kernel NULL pointer dereference at           (null)
[ 5954.093806] IP: [<ffffffff810ba1c8>] __bfs+0xc8/0x260
[ 5954.094331] PGD 1084e5067 PUD 102368067 PMD 0 
[ 5954.094830] Oops: 0000 [#1] SMP 
[ 5954.095194] last sysfs file: /sys/devices/system/cpu/online
[ 5954.095760] CPU 6 
[ 5954.095954] Modules linked in:
[ 5954.096319] 
[ 5954.096483] Pid: 2927, comm: rm Not tainted 2.6.37-rc4-dgc+ #794 /Bochs
[ 5954.096665] RIP: 0010:[<ffffffff810ba1c8>]  [<ffffffff810ba1c8>] __bfs+0xc8/0x260
[ 5954.096665] RSP: 0018:ffff8801175539a8  EFLAGS: 00010046
[ 5954.096665] RAX: ffffffff8267d980 RBX: ffffffff8267d980 RCX: ffff880117553a48
[ 5954.096665] RDX: ffff8801175539d0 RSI: 0000000000000000 RDI: ffff880117553a48
[ 5954.096665] RBP: ffff880117553a08 R08: 0000000000000000 R09: 0000000000000000
[ 5954.096665] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000000
[ 5954.096665] R13: ffffffff810b8e20 R14: ffff880117553a90 R15: 0000000000000000
[ 5954.096665] FS:  00007f4594cf3700(0000) GS:ffff8800dfa00000(0000) knlGS:0000000000000000
[ 5954.096665] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 5954.096665] CR2: 00007f2f21e89c60 CR3: 0000000110b0f000 CR4: 00000000000006e0
[ 5954.096665] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 5954.096665] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 5954.096665] Process rm (pid: 2927, threadinfo ffff880117552000, task ffff88010ff10b00)
[ 5954.096665] Stack:
[ 5954.096665]  ffffffff8267d868 00007fffa02d1428 ffff8800ffffffff ffff880100000000
[ 5954.096665]  000000000000b720 ffff880117553a48 ffffffff8267d868 ffff880117553a48
[ 5954.096665]  0000000000000000 ffff88010ff10b00 0000000000000000 ffffffff81dacba0
[ 5954.096665] Call Trace:
[ 5954.096665]  [<ffffffff810ba4d2>] find_usage_backwards+0x42/0x80
[ 5954.096665]  [<ffffffff810bcec4>] check_usage_backwards+0x64/0xf0
[ 5954.096665]  [<ffffffff8104796f>] ? save_stack_trace+0x2f/0x50
[ 5954.096665]  [<ffffffff810bce60>] ? check_usage_backwards+0x0/0xf0
[ 5954.096665]  [<ffffffff810bd9a9>] mark_lock+0x1a9/0x440
[ 5954.096665]  [<ffffffff810be989>] __lock_acquire+0x5a9/0x14b0
[ 5954.096665]  [<ffffffff810be716>] ? __lock_acquire+0x336/0x14b0
[ 5954.096665]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.096665]  [<ffffffff810bf944>] lock_acquire+0xb4/0x140
[ 5954.096665]  [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.096665]  [<ffffffff81b19d86>] _raw_spin_lock+0x36/0x70
[ 5954.096665]  [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.096665]  [<ffffffff81177a1c>] shrink_dentry_list+0x5c/0x430
[ 5954.096665]  [<ffffffff811779c0>] ? shrink_dentry_list+0x0/0x430
[ 5954.096665]  [<ffffffff816b9c7e>] ? do_raw_spin_unlock+0x5e/0xb0
[ 5954.096665]  [<ffffffff81177f2d>] __shrink_dcache_sb+0x13d/0x1c0
[ 5954.096665]  [<ffffffff811784bf>] shrink_dcache_parent+0x32f/0x390
[ 5954.096665]  [<ffffffff8116d31d>] dentry_unhash+0x3d/0x70
[ 5954.096665]  [<ffffffff8116d3b0>] vfs_rmdir+0x60/0xe0
[ 5954.096665]  [<ffffffff8116f673>] do_rmdir+0x113/0x130
[ 5954.096665]  [<ffffffff8103a03a>] ? sysret_check+0x2e/0x69
[ 5954.096665]  [<ffffffff81b19ae9>] ? trace_hardirqs_on_thunk+0x3a/0x3f
[ 5954.096665]  [<ffffffff8116f6c5>] sys_unlinkat+0x35/0x40
[ 5954.096665]  [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
[ 5954.096665] Code: 0a 89 05 dc 0f a8 01 48 8b 41 10 48 85 c0 0f 84 1f 01 00 00 48 8d 98 70 01 00 00 48 05 80 01 00 00 45 85 c0 48 0f 44 d8 4c 8b 3b <49> 8b 07 49 39  
[ 5954.096665] RIP  [<ffffffff810ba1c8>] __bfs+0xc8/0x260
[ 5954.096665]  RSP <ffff8801175539a8>
[ 5954.096665] CR2: 0000000000000000
[ 5954.127991] ---[ end trace 85a6727c2d4e3d90 ]---

Cheers,

Dave.

Signature · 2 lines
Dave Chinner
[email protected]
reviewer
#85 lore raw
Quoted 7 lines · Dave Chinner
Quoted 6 lines · Nick Piggin

git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.git vfs-scale-working

Here is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:

Quoted 8 lines
Quoted 5 lines

* preparation patches
* introduce new locks to take over dcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking steps

Just got this set of traces doing an 8-way parallel remove of 50 million inodes at about 40M inodes unlinked:

Thanks for testing...

Quoted 41 lines
[ 5954.061633] BUG: sleeping function called from invalid context at arch/x86/mm/fault.c:1081
[ 5954.062466] in_atomic(): 0, irqs_disabled(): 1, pid: 2927, name: rm
[ 5954.063122] 3 locks held by rm/2927:
[ 5954.063476]  #0:  (&sb->s_type->i_mutex_key#12/1){+.+.+.}, at: [<ffffffff8116f5e1>] do_rmdir+0x81/0x130
[ 5954.064014]  #1:  (&sb->s_type->i_mutex_key#12){+.+.+.}, at: [<ffffffff8116d3a8>] vfs_rmdir+0x58/0xe0
[ 5954.064014]  #2:  (rcu_read_lock){.+.+..}, at: [<ffffffff811779c0>] shrink_dentry_list+0x0/0x430
[ 5954.064014] irq event stamp: 1484376719
[ 5954.064014] hardirqs last  enabled at (1484376719): [<ffffffff810ebf07>] __call_rcu+0xd7/0x1a0
[ 5954.064014] hardirqs last disabled at (1484376718): [<ffffffff810ebe7a>] __call_rcu+0x4a/0x1a0
[ 5954.064014] softirqs last  enabled at (1484376586): [<ffffffff8108b911>] __do_softirq+0x161/0x270
[ 5954.064014] softirqs last disabled at (1484376581): [<ffffffff8103af1c>] call_softirq+0x1c/0x50
[ 5954.064014] Pid: 2927, comm: rm Not tainted 2.6.37-rc4-dgc+ #794
[ 5954.064014] Call Trace:
[ 5954.064014]  [<ffffffff810b95b0>] ? print_irqtrace_events+0xd0/0xe0
[ 5954.064014]  [<ffffffff81076455>] __might_sleep+0xf5/0x130
[ 5954.064014]  [<ffffffff81b1e603>] do_page_fault+0x103/0x4f0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff81b19b28>] ? trace_hardirqs_off_thunk+0x3a/0x3c
[ 5954.064014]  [<ffffffff810b8e20>] ? usage_match+0x0/0x20
[ 5954.064014]  [<ffffffff81b1af25>] page_fault+0x25/0x30
[ 5954.064014]  [<ffffffff810b8e20>] ? usage_match+0x0/0x20
[ 5954.064014]  [<ffffffff810ba1c8>] ? __bfs+0xc8/0x260
[ 5954.064014]  [<ffffffff810ba123>] ? __bfs+0x23/0x260
[ 5954.064014]  [<ffffffff810ba4d2>] find_usage_backwards+0x42/0x80
[ 5954.064014]  [<ffffffff810bcec4>] check_usage_backwards+0x64/0xf0
[ 5954.064014]  [<ffffffff8104796f>] ? save_stack_trace+0x2f/0x50
[ 5954.064014]  [<ffffffff810bce60>] ? check_usage_backwards+0x0/0xf0
[ 5954.064014]  [<ffffffff810bd9a9>] mark_lock+0x1a9/0x440
[ 5954.064014]  [<ffffffff810be989>] __lock_acquire+0x5a9/0x14b0
[ 5954.064014]  [<ffffffff810be716>] ? __lock_acquire+0x336/0x14b0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff810bf944>] lock_acquire+0xb4/0x140
[ 5954.064014]  [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.064014]  [<ffffffff81b19d86>] _raw_spin_lock+0x36/0x70
[ 5954.064014]  [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.064014]  [<ffffffff81177a1c>] shrink_dentry_list+0x5c/0x430
[ 5954.064014]  [<ffffffff811779c0>] ? shrink_dentry_list+0x0/0x430
[ 5954.064014]  [<ffffffff816b9c7e>] ? do_raw_spin_unlock+0x5e/0xb0
[ 5954.064014]  [<ffffffff81177f2d>] __shrink_dcache_sb+0x13d/0x1c0
Quoted 8 lines
[ 5954.064014]  [<ffffffff811784bf>] shrink_dcache_parent+0x32f/0x390
[ 5954.064014]  [<ffffffff8116d31d>] dentry_unhash+0x3d/0x70
[ 5954.064014]  [<ffffffff8116d3b0>] vfs_rmdir+0x60/0xe0
[ 5954.064014]  [<ffffffff8116f673>] do_rmdir+0x113/0x130
[ 5954.064014]  [<ffffffff8103a03a>] ? sysret_check+0x2e/0x69
[ 5954.064014]  [<ffffffff81b19ae9>] ? trace_hardirqs_on_thunk+0x3a/0x3f
[ 5954.064014]  [<ffffffff8116f6c5>] sys_unlinkat+0x35/0x40
[ 5954.064014]  [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b

Seems that lockdep exploded.

Quoted 48 lines
[ 5954.092916] BUG: unable to handle kernel NULL pointer dereference at           (null)
[ 5954.093806] IP: [<ffffffff810ba1c8>] __bfs+0xc8/0x260
[ 5954.094331] PGD 1084e5067 PUD 102368067 PMD 0
[ 5954.094830] Oops: 0000 [#1] SMP
[ 5954.095194] last sysfs file: /sys/devices/system/cpu/online
[ 5954.095760] CPU 6
[ 5954.095954] Modules linked in:
[ 5954.096319]
[ 5954.096483] Pid: 2927, comm: rm Not tainted 2.6.37-rc4-dgc+ #794 /Bochs
[ 5954.096665] RIP: 0010:[<ffffffff810ba1c8>]  [<ffffffff810ba1c8>] __bfs+0xc8/0x260
[ 5954.096665] RSP: 0018:ffff8801175539a8  EFLAGS: 00010046
[ 5954.096665] RAX: ffffffff8267d980 RBX: ffffffff8267d980 RCX: ffff880117553a48
[ 5954.096665] RDX: ffff8801175539d0 RSI: 0000000000000000 RDI: ffff880117553a48
[ 5954.096665] RBP: ffff880117553a08 R08: 0000000000000000 R09: 0000000000000000
[ 5954.096665] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000000
[ 5954.096665] R13: ffffffff810b8e20 R14: ffff880117553a90 R15: 0000000000000000
[ 5954.096665] FS:  00007f4594cf3700(0000) GS:ffff8800dfa00000(0000) knlGS:0000000000000000
[ 5954.096665] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 5954.096665] CR2: 00007f2f21e89c60 CR3: 0000000110b0f000 CR4: 00000000000006e0
[ 5954.096665] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 5954.096665] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 5954.096665] Process rm (pid: 2927, threadinfo ffff880117552000, task ffff88010ff10b00)
[ 5954.096665] Stack:
[ 5954.096665]  ffffffff8267d868 00007fffa02d1428 ffff8800ffffffff ffff880100000000
[ 5954.096665]  000000000000b720 ffff880117553a48 ffffffff8267d868 ffff880117553a48
[ 5954.096665]  0000000000000000 ffff88010ff10b00 0000000000000000 ffffffff81dacba0
[ 5954.096665] Call Trace:
[ 5954.096665]  [<ffffffff810ba4d2>] find_usage_backwards+0x42/0x80
[ 5954.096665]  [<ffffffff810bcec4>] check_usage_backwards+0x64/0xf0
[ 5954.096665]  [<ffffffff8104796f>] ? save_stack_trace+0x2f/0x50
[ 5954.096665]  [<ffffffff810bce60>] ? check_usage_backwards+0x0/0xf0
[ 5954.096665]  [<ffffffff810bd9a9>] mark_lock+0x1a9/0x440
[ 5954.096665]  [<ffffffff810be989>] __lock_acquire+0x5a9/0x14b0
[ 5954.096665]  [<ffffffff810be716>] ? __lock_acquire+0x336/0x14b0
[ 5954.096665]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.096665]  [<ffffffff810bf944>] lock_acquire+0xb4/0x140
[ 5954.096665]  [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.096665]  [<ffffffff81b19d86>] _raw_spin_lock+0x36/0x70
[ 5954.096665]  [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.096665]  [<ffffffff81177a1c>] shrink_dentry_list+0x5c/0x430
[ 5954.096665]  [<ffffffff811779c0>] ? shrink_dentry_list+0x0/0x430
[ 5954.096665]  [<ffffffff816b9c7e>] ? do_raw_spin_unlock+0x5e/0xb0
[ 5954.096665]  [<ffffffff81177f2d>] __shrink_dcache_sb+0x13d/0x1c0
[ 5954.096665]  [<ffffffff811784bf>] shrink_dcache_parent+0x32f/0x390
[ 5954.096665]  [<ffffffff8116d31d>] dentry_unhash+0x3d/0x70
[ 5954.096665]  [<ffffffff8116d3b0>] vfs_rmdir+0x60/0xe0
[ 5954.096665]  [<ffffffff8116f673>] do_rmdir+0x113/0x130
[ 5954.096665]  [<ffffffff8103a03a>] ? sysret_check+0x2e/0x69
Quoted 8 lines
[ 5954.096665]  [<ffffffff81b19ae9>] ? trace_hardirqs_on_thunk+0x3a/0x3f
[ 5954.096665]  [<ffffffff8116f6c5>] sys_unlinkat+0x35/0x40
[ 5954.096665]  [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
[ 5954.096665] Code: 0a 89 05 dc 0f a8 01 48 8b 41 10 48 85 c0 0f 84 1f 01 00 00 48 8d 98 70 01 00 00 48 05 80 01 00 00 45 85 c0 48 0f 44 d8 4c 8b 3b <49> 8b 07 49 39
[ 5954.096665] RIP  [<ffffffff810ba1c8>] __bfs+0xc8/0x260
[ 5954.096665]  RSP <ffff8801175539a8>
[ 5954.096665] CR2: 0000000000000000
[ 5954.127991] ---[ end trace 85a6727c2d4e3d90 ]---

So I vfs-scale-working branch may not be entirely in the clear, seeing as it touches
the code lower in the call chain. However I don't know what can cause lockdep to go off
the rails like this.

There is a sequence I used to hack around lockdep nesting restrictions, following this
pattern:

repeat:

    spin_lock(&parent->d_lock);
    spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
    /* do stuff */
    spin_unlock(&parent->d_lock);
    spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
    parent = dentry;
    spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
    goto repeat;

It's not directly in this call chain, but I wonder if it could have caused any problem?
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#86 lore raw
Quoted 10 lines · Nick Piggin

repeat:

    spin_lock(&parent->d_lock);
    spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
    /* do stuff */
    spin_unlock(&parent->d_lock);
    spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
    parent = dentry;
    spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
    goto repeat; 

shouldn't that be s/this_parent/parent/ ?

So what you're trying to do is:

A -> B -> C -> ...

lock A
lock B, nested
unlock A
flip B from nested to top
lock C, nested
unlock B
flip C from nested to top
lock ...

Anyway, the way to write that is something like:

  lock_set_subclass(&detry->d_lock.dep_map, 0, _RET_IP_);

Which will reset the subclass of the held lock from DENTRY_D_LOCK_NESTED to 0.

This is also used in double_unlock_balance(), we go into double_lock_balance() with this_rq locked and want to lock busiest, because of the lock ordering we might need to unlock this_rq and lock busiest first, at which point this_rq is nested.

On unlock we thus need to map this_rq back to subclass 0 (which it had before double_lock_balance(), because otherwise subsequent lock operations will be done against the subclass and confuse things.

reviewer
#87 lore raw
Quoted 12 lines · Peter Zijlstra
Quoted 10 lines · Nick Piggin

 repeat:
    spin_lock(&parent->d_lock); spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); /* do stuff */
    spin_unlock(&parent->d_lock); spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_); parent = dentry; spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_); goto repeat;

Quoted 1 line

shouldn't that be s/this_parent/parent/ ?

Yes, typo in my pseudo code.

Quoted 13 lines

So what you're trying to do is:

 A -> B -> C -> ...

lock A
lock B, nested
unlock A
flip B from nested to top
lock C, nested
unlock B
flip C from nested to top
lock ...

Quoted 6 lines

Anyway, the way to write that is something like:

 lock_set_subclass(&detry->d_lock.dep_map, 0, _RET_IP_);

Which will reset the subclass of the held lock from DENTRY_D_LOCK_NESTED to 0.

OK, thanks. My version should not have caused any problems though, right? Any idea what might have caused Dave's crash?

Quoted 8 lines

This is also used in double_unlock_balance(), we go into double_lock_balance() with this_rq locked and want to lock busiest, because of the lock ordering we might need to unlock this_rq and lock busiest first, at which point this_rq is nested.

On unlock we thus need to map this_rq back to subclass 0 (which it had before double_lock_balance(), because otherwise subsequent lock operations will be done against the subclass and confuse things.

Thanks,
Nick

#88 lore raw
Quoted 2 lines · Nick Piggin

OK, thanks. My version should not have caused any problems though, right?

I tihnk so, yes, altough looking at it again I wonder why you use spin_aquire(.trylock=1) -- but that too shouldn't cause anything like the explosion.

Quoted 1 line

Any idea what might have caused Dave's crash?

Not directly, no. Usually lockdep crashes indicate use after free like things, where we try to lock a lock that's been scribbled on. But that usually explodes a bit earlier.

You faulting in the middle of that breath-first-search does suggest some data corruption, but I'm not quite sure what kind, I can't remember ever having seem something like this before.

I've CC'ed Ming Lei who wrote the bfs search, maybe he's got an idea.

Copy of the splat:
---

Quoted 49 lines
[ 5954.061633] BUG: sleeping function called from invalid context at arch/x86/mm/fault.c:1081
[ 5954.062466] in_atomic(): 0, irqs_disabled(): 1, pid: 2927, name: rm
[ 5954.063122] 3 locks held by rm/2927:
[ 5954.063476]  #0:  (&sb->s_type->i_mutex_key#12/1){+.+.+.}, at: [<ffffffff8116f5e1>] do_rmdir+0x81/0x130
[ 5954.064014]  #1:  (&sb->s_type->i_mutex_key#12){+.+.+.}, at: [<ffffffff8116d3a8>] vfs_rmdir+0x58/0xe0
[ 5954.064014]  #2:  (rcu_read_lock){.+.+..}, at: [<ffffffff811779c0>] shrink_dentry_list+0x0/0x430
[ 5954.064014] irq event stamp: 1484376719
[ 5954.064014] hardirqs last  enabled at (1484376719): [<ffffffff810ebf07>] __call_rcu+0xd7/0x1a0
[ 5954.064014] hardirqs last disabled at (1484376718): [<ffffffff810ebe7a>] __call_rcu+0x4a/0x1a0
[ 5954.064014] softirqs last  enabled at (1484376586): [<ffffffff8108b911>] __do_softirq+0x161/0x270
[ 5954.064014] softirqs last disabled at (1484376581): [<ffffffff8103af1c>] call_softirq+0x1c/0x50
[ 5954.064014] Pid: 2927, comm: rm Not tainted 2.6.37-rc4-dgc+ #794
[ 5954.064014] Call Trace:
[ 5954.064014]  [<ffffffff810b95b0>] ? print_irqtrace_events+0xd0/0xe0
[ 5954.064014]  [<ffffffff81076455>] __might_sleep+0xf5/0x130
[ 5954.064014]  [<ffffffff81b1e603>] do_page_fault+0x103/0x4f0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff81b19b28>] ? trace_hardirqs_off_thunk+0x3a/0x3c
[ 5954.064014]  [<ffffffff810b8e20>] ? usage_match+0x0/0x20
[ 5954.064014]  [<ffffffff81b1af25>] page_fault+0x25/0x30
[ 5954.064014]  [<ffffffff810b8e20>] ? usage_match+0x0/0x20
[ 5954.064014]  [<ffffffff810ba1c8>] ? __bfs+0xc8/0x260
[ 5954.064014]  [<ffffffff810ba123>] ? __bfs+0x23/0x260
[ 5954.064014]  [<ffffffff810ba4d2>] find_usage_backwards+0x42/0x80
[ 5954.064014]  [<ffffffff810bcec4>] check_usage_backwards+0x64/0xf0
[ 5954.064014]  [<ffffffff8104796f>] ? save_stack_trace+0x2f/0x50
[ 5954.064014]  [<ffffffff810bce60>] ? check_usage_backwards+0x0/0xf0
[ 5954.064014]  [<ffffffff810bd9a9>] mark_lock+0x1a9/0x440
[ 5954.064014]  [<ffffffff810be989>] __lock_acquire+0x5a9/0x14b0
[ 5954.064014]  [<ffffffff810be716>] ? __lock_acquire+0x336/0x14b0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff810bf944>] lock_acquire+0xb4/0x140
[ 5954.064014]  [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.064014]  [<ffffffff81b19d86>] _raw_spin_lock+0x36/0x70
[ 5954.064014]  [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.064014]  [<ffffffff81177a1c>] shrink_dentry_list+0x5c/0x430
[ 5954.064014]  [<ffffffff811779c0>] ? shrink_dentry_list+0x0/0x430
[ 5954.064014]  [<ffffffff816b9c7e>] ? do_raw_spin_unlock+0x5e/0xb0
[ 5954.064014]  [<ffffffff81177f2d>] __shrink_dcache_sb+0x13d/0x1c0
[ 5954.064014]  [<ffffffff811784bf>] shrink_dcache_parent+0x32f/0x390
[ 5954.064014]  [<ffffffff8116d31d>] dentry_unhash+0x3d/0x70
[ 5954.064014]  [<ffffffff8116d3b0>] vfs_rmdir+0x60/0xe0
[ 5954.064014]  [<ffffffff8116f673>] do_rmdir+0x113/0x130
[ 5954.064014]  [<ffffffff8103a03a>] ? sysret_check+0x2e/0x69
[ 5954.064014]  [<ffffffff81b19ae9>] ? trace_hardirqs_on_thunk+0x3a/0x3f
[ 5954.064014]  [<ffffffff8116f6c5>] sys_unlinkat+0x35/0x40
[ 5954.064014]  [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
#89 lore raw
In reply to Nick Piggin
Quoted 110 lines · Nick Piggin
Quoted 41 lines · Dave Chinner
[ 5954.061633] BUG: sleeping function called from invalid context at arch/x86/mm/fault.c:1081
[ 5954.062466] in_atomic(): 0, irqs_disabled(): 1, pid: 2927, name: rm
[ 5954.063122] 3 locks held by rm/2927:
[ 5954.063476]  #0:  (&sb->s_type->i_mutex_key#12/1){+.+.+.}, at: [<ffffffff8116f5e1>] do_rmdir+0x81/0x130
[ 5954.064014]  #1:  (&sb->s_type->i_mutex_key#12){+.+.+.}, at: [<ffffffff8116d3a8>] vfs_rmdir+0x58/0xe0
[ 5954.064014]  #2:  (rcu_read_lock){.+.+..}, at: [<ffffffff811779c0>] shrink_dentry_list+0x0/0x430
[ 5954.064014] irq event stamp: 1484376719
[ 5954.064014] hardirqs last  enabled at (1484376719): [<ffffffff810ebf07>] __call_rcu+0xd7/0x1a0
[ 5954.064014] hardirqs last disabled at (1484376718): [<ffffffff810ebe7a>] __call_rcu+0x4a/0x1a0
[ 5954.064014] softirqs last  enabled at (1484376586): [<ffffffff8108b911>] __do_softirq+0x161/0x270
[ 5954.064014] softirqs last disabled at (1484376581): [<ffffffff8103af1c>] call_softirq+0x1c/0x50
[ 5954.064014] Pid: 2927, comm: rm Not tainted 2.6.37-rc4-dgc+ #794
[ 5954.064014] Call Trace:
[ 5954.064014]  [<ffffffff810b95b0>] ? print_irqtrace_events+0xd0/0xe0
[ 5954.064014]  [<ffffffff81076455>] __might_sleep+0xf5/0x130
[ 5954.064014]  [<ffffffff81b1e603>] do_page_fault+0x103/0x4f0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff81b19b28>] ? trace_hardirqs_off_thunk+0x3a/0x3c
[ 5954.064014]  [<ffffffff810b8e20>] ? usage_match+0x0/0x20
[ 5954.064014]  [<ffffffff81b1af25>] page_fault+0x25/0x30
[ 5954.064014]  [<ffffffff810b8e20>] ? usage_match+0x0/0x20
[ 5954.064014]  [<ffffffff810ba1c8>] ? __bfs+0xc8/0x260
[ 5954.064014]  [<ffffffff810ba123>] ? __bfs+0x23/0x260
[ 5954.064014]  [<ffffffff810ba4d2>] find_usage_backwards+0x42/0x80
[ 5954.064014]  [<ffffffff810bcec4>] check_usage_backwards+0x64/0xf0
[ 5954.064014]  [<ffffffff8104796f>] ? save_stack_trace+0x2f/0x50
[ 5954.064014]  [<ffffffff810bce60>] ? check_usage_backwards+0x0/0xf0
[ 5954.064014]  [<ffffffff810bd9a9>] mark_lock+0x1a9/0x440
[ 5954.064014]  [<ffffffff810be989>] __lock_acquire+0x5a9/0x14b0
[ 5954.064014]  [<ffffffff810be716>] ? __lock_acquire+0x336/0x14b0
[ 5954.064014]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014]  [<ffffffff810bf944>] lock_acquire+0xb4/0x140
[ 5954.064014]  [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.064014]  [<ffffffff81b19d86>] _raw_spin_lock+0x36/0x70
[ 5954.064014]  [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.064014]  [<ffffffff81177a1c>] shrink_dentry_list+0x5c/0x430
[ 5954.064014]  [<ffffffff811779c0>] ? shrink_dentry_list+0x0/0x430
[ 5954.064014]  [<ffffffff816b9c7e>] ? do_raw_spin_unlock+0x5e/0xb0
[ 5954.064014]  [<ffffffff81177f2d>] __shrink_dcache_sb+0x13d/0x1c0
Quoted 8 lines
[ 5954.064014]  [<ffffffff811784bf>] shrink_dcache_parent+0x32f/0x390
[ 5954.064014]  [<ffffffff8116d31d>] dentry_unhash+0x3d/0x70
[ 5954.064014]  [<ffffffff8116d3b0>] vfs_rmdir+0x60/0xe0
[ 5954.064014]  [<ffffffff8116f673>] do_rmdir+0x113/0x130
[ 5954.064014]  [<ffffffff8103a03a>] ? sysret_check+0x2e/0x69
[ 5954.064014]  [<ffffffff81b19ae9>] ? trace_hardirqs_on_thunk+0x3a/0x3f
[ 5954.064014]  [<ffffffff8116f6c5>] sys_unlinkat+0x35/0x40
[ 5954.064014]  [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b

Seems that lockdep exploded.

Quoted 48 lines
[ 5954.092916] BUG: unable to handle kernel NULL pointer dereference at           (null)
[ 5954.093806] IP: [<ffffffff810ba1c8>] __bfs+0xc8/0x260
[ 5954.094331] PGD 1084e5067 PUD 102368067 PMD 0
[ 5954.094830] Oops: 0000 [#1] SMP
[ 5954.095194] last sysfs file: /sys/devices/system/cpu/online
[ 5954.095760] CPU 6
[ 5954.095954] Modules linked in:
[ 5954.096319]
[ 5954.096483] Pid: 2927, comm: rm Not tainted 2.6.37-rc4-dgc+ #794 /Bochs
[ 5954.096665] RIP: 0010:[<ffffffff810ba1c8>]  [<ffffffff810ba1c8>] __bfs+0xc8/0x260
[ 5954.096665] RSP: 0018:ffff8801175539a8  EFLAGS: 00010046
[ 5954.096665] RAX: ffffffff8267d980 RBX: ffffffff8267d980 RCX: ffff880117553a48
[ 5954.096665] RDX: ffff8801175539d0 RSI: 0000000000000000 RDI: ffff880117553a48
[ 5954.096665] RBP: ffff880117553a08 R08: 0000000000000000 R09: 0000000000000000
[ 5954.096665] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000000
[ 5954.096665] R13: ffffffff810b8e20 R14: ffff880117553a90 R15: 0000000000000000
[ 5954.096665] FS:  00007f4594cf3700(0000) GS:ffff8800dfa00000(0000) knlGS:0000000000000000
[ 5954.096665] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 5954.096665] CR2: 00007f2f21e89c60 CR3: 0000000110b0f000 CR4: 00000000000006e0
[ 5954.096665] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 5954.096665] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 5954.096665] Process rm (pid: 2927, threadinfo ffff880117552000, task ffff88010ff10b00)
[ 5954.096665] Stack:
[ 5954.096665]  ffffffff8267d868 00007fffa02d1428 ffff8800ffffffff ffff880100000000
[ 5954.096665]  000000000000b720 ffff880117553a48 ffffffff8267d868 ffff880117553a48
[ 5954.096665]  0000000000000000 ffff88010ff10b00 0000000000000000 ffffffff81dacba0
[ 5954.096665] Call Trace:
[ 5954.096665]  [<ffffffff810ba4d2>] find_usage_backwards+0x42/0x80
[ 5954.096665]  [<ffffffff810bcec4>] check_usage_backwards+0x64/0xf0
[ 5954.096665]  [<ffffffff8104796f>] ? save_stack_trace+0x2f/0x50
[ 5954.096665]  [<ffffffff810bce60>] ? check_usage_backwards+0x0/0xf0
[ 5954.096665]  [<ffffffff810bd9a9>] mark_lock+0x1a9/0x440
[ 5954.096665]  [<ffffffff810be989>] __lock_acquire+0x5a9/0x14b0
[ 5954.096665]  [<ffffffff810be716>] ? __lock_acquire+0x336/0x14b0
[ 5954.096665]  [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.096665]  [<ffffffff810bf944>] lock_acquire+0xb4/0x140
[ 5954.096665]  [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.096665]  [<ffffffff81b19d86>] _raw_spin_lock+0x36/0x70
[ 5954.096665]  [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.096665]  [<ffffffff81177a1c>] shrink_dentry_list+0x5c/0x430
[ 5954.096665]  [<ffffffff811779c0>] ? shrink_dentry_list+0x0/0x430
[ 5954.096665]  [<ffffffff816b9c7e>] ? do_raw_spin_unlock+0x5e/0xb0
[ 5954.096665]  [<ffffffff81177f2d>] __shrink_dcache_sb+0x13d/0x1c0
[ 5954.096665]  [<ffffffff811784bf>] shrink_dcache_parent+0x32f/0x390
[ 5954.096665]  [<ffffffff8116d31d>] dentry_unhash+0x3d/0x70
[ 5954.096665]  [<ffffffff8116d3b0>] vfs_rmdir+0x60/0xe0
[ 5954.096665]  [<ffffffff8116f673>] do_rmdir+0x113/0x130
[ 5954.096665]  [<ffffffff8103a03a>] ? sysret_check+0x2e/0x69
Quoted 8 lines
[ 5954.096665]  [<ffffffff81b19ae9>] ? trace_hardirqs_on_thunk+0x3a/0x3f
[ 5954.096665]  [<ffffffff8116f6c5>] sys_unlinkat+0x35/0x40
[ 5954.096665]  [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
[ 5954.096665] Code: 0a 89 05 dc 0f a8 01 48 8b 41 10 48 85 c0 0f 84 1f 01 00 00 48 8d 98 70 01 00 00 48 05 80 01 00 00 45 85 c0 48 0f 44 d8 4c 8b 3b <49> 8b 07 49 39
[ 5954.096665] RIP  [<ffffffff810ba1c8>] __bfs+0xc8/0x260
[ 5954.096665]  RSP <ffff8801175539a8>
[ 5954.096665] CR2: 0000000000000000
[ 5954.127991] ---[ end trace 85a6727c2d4e3d90 ]---
Quoted 1 line

So vfs-scale-working branch may not be entirely in the clear, seeing

Ah, may have been a stupid little bug. The list entry check was being done and then the pointer reloaded to be used.

What does the asm for shrink_dentry_list look like (before this patch)?

Thanks,
Nick

fs/dcache.c +4 -4
Index: linux-2.6/fs/dcache.c
===================================================================
@@ -657,10 +657,10 @@ static void shrink_dentry_list(struct li
 	struct dentry *dentry;
 
 	rcu_read_lock();
-	while (!list_empty(list)) {
-		dentry = list_entry(list->prev, struct dentry, d_lru);
-
-		/* Don't need RCU dereference because we recheck under lock */
+	for (;;) {
+		dentry = list_entry_rcu(list->prev, struct dentry, d_lru);
+		if (&dentry->d_lru == list)
+			break;
 		spin_lock(&dentry->d_lock);
 		if (dentry != list_entry(list->prev, struct dentry, d_lru)) {
 			spin_unlock(&dentry->d_lock);
#90 lore raw
In reply to Nick Piggin
Quoted 11 lines · Nick Piggin

git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.git vfs-scale-working

Here is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:

* preparation patches
* introduce new locks to take over dcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking steps

Stress test doing:

single thread 50M inode create
single thread rm -rf
2-way 50M inode create
2-way rm -rf
4-way 50M inode create
4-way rm -rf
8-way 50M inode create
8-way rm -rf
8-way 250M inode create
8-way rm -rf

Failed about 5 minutes into the "4-way rm -rf" (~3 hours into the test) with a CPU stuck spinning on here:

[37372.084012] NMI backtrace for cpu 5
[37372.084012] CPU 5 
[37372.084012] Modules linked in:
[37372.084012] 
[37372.084012] Pid: 15214, comm: rm Not tainted 2.6.37-rc4-dgc+ #797 /Bochs
[37372.084012] RIP: 0010:[<ffffffff810643c4>]  [<ffffffff810643c4>] __ticket_spin_lock+0x14/0x20
[37372.084012] RSP: 0018:ffff880114643c98  EFLAGS: 00000213
[37372.084012] RAX: 0000000000008801 RBX: ffff8800687be6c0 RCX: ffff8800c4eb2688
[37372.084012] RDX: ffff880114643d38 RSI: ffff8800dfd4ea80 RDI: ffff880114643d14
[37372.084012] RBP: ffff880114643c98 R08: 0000000000000003 R09: 0000000000000000
[37372.084012] R10: 0000000000000000 R11: dead000000200200 R12: ffff880114643d14
[37372.084012] R13: ffff880114643cb8 R14: ffff880114643d38 R15: ffff8800687be71c
[37372.084012] FS:  00007fd6d7c93700(0000) GS:ffff8800dfd40000(0000) knlGS:0000000000000000
[37372.084012] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[37372.084012] CR2: 0000000000bbd108 CR3: 0000000107146000 CR4: 00000000000006e0
[37372.084012] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[37372.084012] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[37372.084012] Process rm (pid: 15214, threadinfo ffff880114642000, task ffff88011b16f890)
[37372.084012] Stack:
[37372.084012]  ffff880114643ca8 ffffffff81ad044e ffff880114643cf8 ffffffff81167ae7
[37372.084012]  0000000000000000 ffff880114643d38 000000000000000e ffff88011901d800
[37372.084012]  ffff8800cdb7cf5c ffff88011901d8e0 0000000000000000 0000000000000000
[37372.084012] Call Trace:
[37372.084012]  [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370
[37372.084012]  [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0
[37372.084012]  [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0
[37372.084012]  [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff8115daa2>] dentry_unhash+0x42/0x80
[37372.084012]  [<ffffffff8115db48>] vfs_rmdir+0x68/0x100
[37372.084012]  [<ffffffff8115fd93>] do_rmdir+0x113/0x130
[37372.084012]  [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90
[37372.084012]  [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40
[37372.084012]  [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
[37372.084012] Code: c1 c1 41 06 81 e9 dd fe ff ff 90 90 90 90 90 90 90 90 90 90 90 90 90 55 b8 00 01 00 00 48 89 e5 f0 66 0f c1 07 38 e0 74 06 f3 90 <8a> 07 eb f6 c9 c3 66 0f 1f 44 00 00 55 48 89 e5 0f b7 07 38 e0 
[37372.084012] Call Trace:
[37372.084012]  [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370
[37372.084012]  [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0
[37372.084012]  [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0
[37372.084012]  [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff8115daa2>] dentry_unhash+0x42/0x80
[37372.084012]  [<ffffffff8115db48>] vfs_rmdir+0x68/0x100
[37372.084012]  [<ffffffff8115fd93>] do_rmdir+0x113/0x130
[37372.084012]  [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90
[37372.084012]  [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40
[37372.084012]  [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b

Cheers,

Dave.

Signature · 2 lines
Dave Chinner
[email protected]
reviewer
#91 lore raw
Quoted 55 lines · Dave Chinner
Quoted 11 lines · Nick Piggin

git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.git vfs-scale-working

Here is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:

* preparation patches
* introduce new locks to take over dcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking steps

Stress test doing:

       single thread 50M inode create
       single thread rm -rf
       2-way 50M inode create
       2-way rm -rf
       4-way 50M inode create
       4-way rm -rf
       8-way 50M inode create
       8-way rm -rf
       8-way 250M inode create
       8-way rm -rf

Failed about 5 minutes into the "4-way rm -rf" (~3 hours into the test) with a CPU stuck spinning on here:

[37372.084012] NMI backtrace for cpu 5
[37372.084012] CPU 5
[37372.084012] Modules linked in:
[37372.084012]
[37372.084012] Pid: 15214, comm: rm Not tainted 2.6.37-rc4-dgc+ #797 /Bochs
[37372.084012] RIP: 0010:[<ffffffff810643c4>]  [<ffffffff810643c4>] __ticket_spin_lock+0x14/0x20
[37372.084012] RSP: 0018:ffff880114643c98  EFLAGS: 00000213
[37372.084012] RAX: 0000000000008801 RBX: ffff8800687be6c0 RCX: ffff8800c4eb2688
[37372.084012] RDX: ffff880114643d38 RSI: ffff8800dfd4ea80 RDI: ffff880114643d14
[37372.084012] RBP: ffff880114643c98 R08: 0000000000000003 R09: 0000000000000000
[37372.084012] R10: 0000000000000000 R11: dead000000200200 R12: ffff880114643d14
[37372.084012] R13: ffff880114643cb8 R14: ffff880114643d38 R15: ffff8800687be71c
[37372.084012] FS:  00007fd6d7c93700(0000) GS:ffff8800dfd40000(0000) knlGS:0000000000000000
[37372.084012] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[37372.084012] CR2: 0000000000bbd108 CR3: 0000000107146000 CR4: 00000000000006e0
[37372.084012] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[37372.084012] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[37372.084012] Process rm (pid: 15214, threadinfo ffff880114642000, task ffff88011b16f890)
[37372.084012] Stack:
[37372.084012]  ffff880114643ca8 ffffffff81ad044e ffff880114643cf8 ffffffff81167ae7
[37372.084012]  0000000000000000 ffff880114643d38 000000000000000e ffff88011901d800
[37372.084012]  ffff8800cdb7cf5c ffff88011901d8e0 0000000000000000 0000000000000000
[37372.084012] Call Trace:
[37372.084012]  [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370
[37372.084012]  [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0
Quoted 8 lines
[37372.084012]  [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0
[37372.084012]  [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff8115daa2>] dentry_unhash+0x42/0x80
[37372.084012]  [<ffffffff8115db48>] vfs_rmdir+0x68/0x100
[37372.084012]  [<ffffffff8115fd93>] do_rmdir+0x113/0x130
[37372.084012]  [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90
[37372.084012]  [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40
[37372.084012]  [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b

OK good, with any luck, that's the same bug.

Is this XFS? Is there any concurrent activity happening on the same dentries? Ie. are the rm -rf threads running on the same directories, or is there any reclaim
happening in the background?

Thanks,
Nick

#92 lore raw
Quoted 65 lines · Nick Piggin
Quoted 55 lines · Dave Chinner
Quoted 11 lines · Nick Piggin

git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.git vfs-scale-working

Here is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:

* preparation patches
* introduce new locks to take over dcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking steps

Stress test doing:

       single thread 50M inode create
       single thread rm -rf
       2-way 50M inode create
       2-way rm -rf
       4-way 50M inode create
       4-way rm -rf
       8-way 50M inode create
       8-way rm -rf
       8-way 250M inode create
       8-way rm -rf

Failed about 5 minutes into the "4-way rm -rf" (~3 hours into the test) with a CPU stuck spinning on here:

[37372.084012] NMI backtrace for cpu 5
[37372.084012] CPU 5
[37372.084012] Modules linked in:
[37372.084012]
[37372.084012] Pid: 15214, comm: rm Not tainted 2.6.37-rc4-dgc+ #797 /Bochs
[37372.084012] RIP: 0010:[<ffffffff810643c4>]  [<ffffffff810643c4>] __ticket_spin_lock+0x14/0x20
[37372.084012] RSP: 0018:ffff880114643c98  EFLAGS: 00000213
[37372.084012] RAX: 0000000000008801 RBX: ffff8800687be6c0 RCX: ffff8800c4eb2688
[37372.084012] RDX: ffff880114643d38 RSI: ffff8800dfd4ea80 RDI: ffff880114643d14
[37372.084012] RBP: ffff880114643c98 R08: 0000000000000003 R09: 0000000000000000
[37372.084012] R10: 0000000000000000 R11: dead000000200200 R12: ffff880114643d14
[37372.084012] R13: ffff880114643cb8 R14: ffff880114643d38 R15: ffff8800687be71c
[37372.084012] FS:  00007fd6d7c93700(0000) GS:ffff8800dfd40000(0000) knlGS:0000000000000000
[37372.084012] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[37372.084012] CR2: 0000000000bbd108 CR3: 0000000107146000 CR4: 00000000000006e0
[37372.084012] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[37372.084012] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[37372.084012] Process rm (pid: 15214, threadinfo ffff880114642000, task ffff88011b16f890)
[37372.084012] Stack:
[37372.084012]  ffff880114643ca8 ffffffff81ad044e ffff880114643cf8 ffffffff81167ae7
[37372.084012]  0000000000000000 ffff880114643d38 000000000000000e ffff88011901d800
[37372.084012]  ffff8800cdb7cf5c ffff88011901d8e0 0000000000000000 0000000000000000
[37372.084012] Call Trace:
[37372.084012]  [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370
[37372.084012]  [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0
Quoted 8 lines
[37372.084012]  [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0
[37372.084012]  [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff8115daa2>] dentry_unhash+0x42/0x80
[37372.084012]  [<ffffffff8115db48>] vfs_rmdir+0x68/0x100
[37372.084012]  [<ffffffff8115fd93>] do_rmdir+0x113/0x130
[37372.084012]  [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90
[37372.084012]  [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40
[37372.084012]  [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
Quoted 3 lines

OK good, with any luck, that's the same bug.

Is this XFS?

Yes.

Quoted 1 line

Is there any concurrent activity happening on the same dentries?

Not from an application perspective.

Quoted 1 line

Ie. are the rm -rf threads running on the same directories,

No, each thread operating on a different directory.

Quoted 1 line

or is there any reclaim happening in the background?

IIRC, kswapd was consuming about 5-10% of a CPU during parallel unlink tests. Mainly reclaiming XFS inodes, I think, but there may be dentry cache reclaim going as well.

Cheers,

Dave.

Signature · 2 lines
Dave Chinner
[email protected]
#93 lore raw
Quoted 78 lines · Dave Chinner
Quoted 65 lines · Nick Piggin
Quoted 55 lines · Dave Chinner
Quoted 11 lines · Nick Piggin

git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.git vfs-scale-working

Here is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:

* preparation patches
* introduce new locks to take over dcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking steps

Stress test doing:

       single thread 50M inode create
       single thread rm -rf
       2-way 50M inode create
       2-way rm -rf
       4-way 50M inode create
       4-way rm -rf
       8-way 50M inode create
       8-way rm -rf
       8-way 250M inode create
       8-way rm -rf

Failed about 5 minutes into the "4-way rm -rf" (~3 hours into the test) with a CPU stuck spinning on here:

[37372.084012] NMI backtrace for cpu 5
[37372.084012] CPU 5
[37372.084012] Modules linked in:
[37372.084012]
[37372.084012] Pid: 15214, comm: rm Not tainted 2.6.37-rc4-dgc+ #797 /Bochs
[37372.084012] RIP: 0010:[<ffffffff810643c4>]  [<ffffffff810643c4>] __ticket_spin_lock+0x14/0x20
[37372.084012] RSP: 0018:ffff880114643c98  EFLAGS: 00000213
[37372.084012] RAX: 0000000000008801 RBX: ffff8800687be6c0 RCX: ffff8800c4eb2688
[37372.084012] RDX: ffff880114643d38 RSI: ffff8800dfd4ea80 RDI: ffff880114643d14
[37372.084012] RBP: ffff880114643c98 R08: 0000000000000003 R09: 0000000000000000
[37372.084012] R10: 0000000000000000 R11: dead000000200200 R12: ffff880114643d14
[37372.084012] R13: ffff880114643cb8 R14: ffff880114643d38 R15: ffff8800687be71c
[37372.084012] FS:  00007fd6d7c93700(0000) GS:ffff8800dfd40000(0000) knlGS:0000000000000000
[37372.084012] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[37372.084012] CR2: 0000000000bbd108 CR3: 0000000107146000 CR4: 00000000000006e0
[37372.084012] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[37372.084012] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[37372.084012] Process rm (pid: 15214, threadinfo ffff880114642000, task ffff88011b16f890)
[37372.084012] Stack:
[37372.084012]  ffff880114643ca8 ffffffff81ad044e ffff880114643cf8 ffffffff81167ae7
[37372.084012]  0000000000000000 ffff880114643d38 000000000000000e ffff88011901d800
[37372.084012]  ffff8800cdb7cf5c ffff88011901d8e0 0000000000000000 0000000000000000
[37372.084012] Call Trace:
[37372.084012]  [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370
[37372.084012]  [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0
Quoted 8 lines
[37372.084012]  [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0
[37372.084012]  [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff8115daa2>] dentry_unhash+0x42/0x80
[37372.084012]  [<ffffffff8115db48>] vfs_rmdir+0x68/0x100
[37372.084012]  [<ffffffff8115fd93>] do_rmdir+0x113/0x130
[37372.084012]  [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90
[37372.084012]  [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40
[37372.084012]  [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
Quoted 3 lines

OK good, with any luck, that's the same bug.

Is this XFS?

Yes.

Quoted 1 line

Is there any concurrent activity happening on the same dentries?

Not from an application perspective.

Quoted 1 line

Ie. are the rm -rf threads running on the same directories,

Quoted 7 lines

No, each thread operating on a different directory.

Quoted 1 line

or is there any reclaim happening in the background?

IIRC, kswapd was consuming about 5-10% of a CPU during parallel unlink tests. Mainly reclaiming XFS inodes, I think, but there may be dentry cache reclaim going as well.

Turns out that the kswapd peaks are upwards of 50% of a CPU for a few seconds, then idle for 10-15s. Typical perf top output of kswapd while it is active during unlinks is:

             samples  pcnt function                    DSO
             _______ _____ ___________________________ _________________

            17168.00 10.2% __call_rcu                  [kernel.kallsyms]
            13223.00  7.8% kmem_cache_free             [kernel.kallsyms]
            12917.00  7.6% down_write                  [kernel.kallsyms]
            12665.00  7.5% xfs_iunlock                 [kernel.kallsyms]
            10493.00  6.2% xfs_reclaim_inode_grab      [kernel.kallsyms]
             9314.00  5.5% __lookup_tag                [kernel.kallsyms]
             9040.00  5.4% radix_tree_delete           [kernel.kallsyms]
             8694.00  5.1% is_bad_inode                [kernel.kallsyms]
             7639.00  4.5% __ticket_spin_lock          [kernel.kallsyms]
             6821.00  4.0% _raw_spin_unlock_irqrestore [kernel.kallsyms]
             5484.00  3.2% __d_drop                    [kernel.kallsyms]
             5114.00  3.0% xfs_reclaim_inode           [kernel.kallsyms]
             4626.00  2.7% __rcu_process_callbacks     [kernel.kallsyms]
             3556.00  2.1% up_write                    [kernel.kallsyms]
             3206.00  1.9% _cond_resched               [kernel.kallsyms]
             3129.00  1.9% xfs_qm_dqdetach             [kernel.kallsyms]
             2327.00  1.4% radix_tree_tag_clear        [kernel.kallsyms]
             2327.00  1.4% call_rcu_sched              [kernel.kallsyms]
             2262.00  1.3% __ticket_spin_unlock        [kernel.kallsyms]
             2215.00  1.3% xfs_ilock                   [kernel.kallsyms]
             2200.00  1.3% radix_tree_gang_lookup_tag  [kernel.kallsyms]
             1982.00  1.2% xfs_reclaim_inodes_ag       [kernel.kallsyms]
             1736.00  1.0% xfs_trans_unlocked_item     [kernel.kallsyms]
             1707.00  1.0% __ticket_spin_trylock       [kernel.kallsyms]
             1688.00  1.0% xfs_perag_get_tag           [kernel.kallsyms]
             1660.00  1.0% flat_send_IPI_mask          [kernel.kallsyms]
             1538.00  0.9% xfs_inode_item_destroy      [kernel.kallsyms]
             1312.00  0.8% __shrink_dcache_sb          [kernel.kallsyms]
              940.00  0.6% xfs_perag_put               [kernel.kallsyms]

So there is some dentry cache reclaim going on.

FWIW, it appears there is quite a lot of RCU freeing overhead (~15% more CPU time) in the work kswapd is doing during these unlinks, too. I just had a look at kswapd when a 8-way create is running - it's running at 50-60% of a cpu for seconds at a time. I caught this while it was doing pure XFS inode cache reclaim (~10s sample, kswapd reclaimed ~1M inodes):

             samples  pcnt function                    DSO
             _______ _____ ___________________________ _________________

            27171.00  9.0% __call_rcu                  [kernel.kallsyms]
            21491.00  7.1% down_write                  [kernel.kallsyms]
            20916.00  6.9% xfs_reclaim_inode           [kernel.kallsyms]
            20313.00  6.7% radix_tree_delete           [kernel.kallsyms]
            15828.00  5.3% kmem_cache_free             [kernel.kallsyms]
            15819.00  5.2% xfs_idestroy_fork           [kernel.kallsyms]
            14893.00  4.9% is_bad_inode                [kernel.kallsyms]
            14666.00  4.9% _raw_spin_unlock_irqrestore [kernel.kallsyms]
            14191.00  4.7% xfs_reclaim_inode_grab      [kernel.kallsyms]
            14105.00  4.7% xfs_iunlock                 [kernel.kallsyms]
            10916.00  3.6% __ticket_spin_lock          [kernel.kallsyms]
            10125.00  3.4% xfs_iflush_cluster          [kernel.kallsyms]
             8221.00  2.7% xfs_qm_dqdetach             [kernel.kallsyms]
             7639.00  2.5% xfs_trans_unlocked_item     [kernel.kallsyms]
             7028.00  2.3% xfs_synchronize_times       [kernel.kallsyms]
             6974.00  2.3% up_write                    [kernel.kallsyms]
             5870.00  1.9% call_rcu_sched              [kernel.kallsyms]
             5634.00  1.9% _cond_resched               [kernel.kallsyms]

Which is showing a similar amount of RCU overhead as the unlink as above. And this while it was doing dentry cache reclaim (~10s sample):

            35921.00 15.7% __d_drop                      [kernel.kallsyms]
            30056.00 13.1% __ticket_spin_trylock         [kernel.kallsyms]
            29066.00 12.7% __ticket_spin_lock            [kernel.kallsyms]
            19043.00  8.3% __call_rcu                    [kernel.kallsyms]
            10098.00  4.4% iput                          [kernel.kallsyms]
             7013.00  3.1% __shrink_dcache_sb            [kernel.kallsyms]
             6774.00  3.0% __percpu_counter_add          [kernel.kallsyms]
             6708.00  2.9% radix_tree_tag_set            [kernel.kallsyms]
             5362.00  2.3% xfs_inactive                  [kernel.kallsyms]
             5130.00  2.2% __ticket_spin_unlock          [kernel.kallsyms]
             4884.00  2.1% call_rcu_sched                [kernel.kallsyms]
             4621.00  2.0% dentry_lru_del                [kernel.kallsyms]
             3735.00  1.6% bit_waitqueue                 [kernel.kallsyms]
             3727.00  1.6% dentry_iput                   [kernel.kallsyms]
             3473.00  1.5% shrink_icache_memory          [kernel.kallsyms]
             3279.00  1.4% kfree                         [kernel.kallsyms]
             3101.00  1.4% xfs_perag_get                 [kernel.kallsyms]
             2516.00  1.1% kmem_cache_free               [kernel.kallsyms]
             2272.00  1.0% shrink_dentry_list            [kernel.kallsyms]

I've never really seen any signficant dentry cache reclaim overhead in profiles of these workloads before, so this was a bit of a surprise....

Cheers,

Dave.

Signature · 2 lines
Dave Chinner
[email protected]
#94 lore raw
Quoted 73 lines · Dave Chinner
Quoted 72 lines · Dave Chinner
Quoted 65 lines · Nick Piggin
Quoted 55 lines · Dave Chinner
Quoted 11 lines · Nick Piggin

git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.git vfs-scale-working

Here is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:

* preparation patches
* introduce new locks to take over dcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking steps

Stress test doing:

       single thread 50M inode create
       single thread rm -rf
       2-way 50M inode create
       2-way rm -rf
       4-way 50M inode create
       4-way rm -rf
       8-way 50M inode create
       8-way rm -rf
       8-way 250M inode create
       8-way rm -rf

Failed about 5 minutes into the "4-way rm -rf" (~3 hours into the test) with a CPU stuck spinning on here:

[37372.084012] NMI backtrace for cpu 5
[37372.084012] CPU 5
[37372.084012] Modules linked in:
[37372.084012]
[37372.084012] Pid: 15214, comm: rm Not tainted 2.6.37-rc4-dgc+ #797 /Bochs
[37372.084012] RIP: 0010:[<ffffffff810643c4>]  [<ffffffff810643c4>] __ticket_spin_lock+0x14/0x20
[37372.084012] RSP: 0018:ffff880114643c98  EFLAGS: 00000213
[37372.084012] RAX: 0000000000008801 RBX: ffff8800687be6c0 RCX: ffff8800c4eb2688
[37372.084012] RDX: ffff880114643d38 RSI: ffff8800dfd4ea80 RDI: ffff880114643d14
[37372.084012] RBP: ffff880114643c98 R08: 0000000000000003 R09: 0000000000000000
[37372.084012] R10: 0000000000000000 R11: dead000000200200 R12: ffff880114643d14
[37372.084012] R13: ffff880114643cb8 R14: ffff880114643d38 R15: ffff8800687be71c
[37372.084012] FS:  00007fd6d7c93700(0000) GS:ffff8800dfd40000(0000) knlGS:0000000000000000
[37372.084012] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[37372.084012] CR2: 0000000000bbd108 CR3: 0000000107146000 CR4: 00000000000006e0
[37372.084012] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[37372.084012] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[37372.084012] Process rm (pid: 15214, threadinfo ffff880114642000, task ffff88011b16f890)
[37372.084012] Stack:
[37372.084012]  ffff880114643ca8 ffffffff81ad044e ffff880114643cf8 ffffffff81167ae7
[37372.084012]  0000000000000000 ffff880114643d38 000000000000000e ffff88011901d800
[37372.084012]  ffff8800cdb7cf5c ffff88011901d8e0 0000000000000000 0000000000000000
[37372.084012] Call Trace:
[37372.084012]  [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370
[37372.084012]  [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0
Quoted 8 lines
[37372.084012]  [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0
[37372.084012]  [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff8115daa2>] dentry_unhash+0x42/0x80
[37372.084012]  [<ffffffff8115db48>] vfs_rmdir+0x68/0x100
[37372.084012]  [<ffffffff8115fd93>] do_rmdir+0x113/0x130
[37372.084012]  [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90
[37372.084012]  [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40
[37372.084012]  [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
Quoted 3 lines

OK good, with any luck, that's the same bug.

Is this XFS?

Yes.

Quoted 7 lines
Quoted 7 lines
Quoted 1 line

Is there any concurrent activity happening on the same dentries?

Not from an application perspective.

Quoted 1 line

Ie. are the rm -rf threads running on the same directories,

No, each thread operating on a different directory.

This is probably fixed by the same patch as the lockdep splat trace.

Quoted 96 lines
Quoted 5 lines
Quoted 1 line

or is there any reclaim happening in the background?

IIRC, kswapd was consuming about 5-10% of a CPU during parallel unlink tests. Mainly reclaiming XFS inodes, I think, but there may be dentry cache reclaim going as well.

Turns out that the kswapd peaks are upwards of 50% of a CPU for a few seconds, then idle for 10-15s. Typical perf top output of kswapd while it is active during unlinks is:

             samples  pcnt function                    DSO
             _______ _____ ___________________________ _________________

            17168.00 10.2% __call_rcu                  [kernel.kallsyms]
            13223.00  7.8% kmem_cache_free             [kernel.kallsyms]
            12917.00  7.6% down_write                  [kernel.kallsyms]
            12665.00  7.5% xfs_iunlock                 [kernel.kallsyms]
            10493.00  6.2% xfs_reclaim_inode_grab      [kernel.kallsyms]
             9314.00  5.5% __lookup_tag                [kernel.kallsyms]
             9040.00  5.4% radix_tree_delete           [kernel.kallsyms]
             8694.00  5.1% is_bad_inode                [kernel.kallsyms]
             7639.00  4.5% __ticket_spin_lock          [kernel.kallsyms]
             6821.00  4.0% _raw_spin_unlock_irqrestore [kernel.kallsyms]
             5484.00  3.2% __d_drop                    [kernel.kallsyms]
             5114.00  3.0% xfs_reclaim_inode           [kernel.kallsyms]
             4626.00  2.7% __rcu_process_callbacks     [kernel.kallsyms]
             3556.00  2.1% up_write                    [kernel.kallsyms]
             3206.00  1.9% _cond_resched               [kernel.kallsyms]
             3129.00  1.9% xfs_qm_dqdetach             [kernel.kallsyms]
             2327.00  1.4% radix_tree_tag_clear        [kernel.kallsyms]
             2327.00  1.4% call_rcu_sched              [kernel.kallsyms]
             2262.00  1.3% __ticket_spin_unlock        [kernel.kallsyms]
             2215.00  1.3% xfs_ilock                   [kernel.kallsyms]
             2200.00  1.3% radix_tree_gang_lookup_tag  [kernel.kallsyms]
             1982.00  1.2% xfs_reclaim_inodes_ag       [kernel.kallsyms]
             1736.00  1.0% xfs_trans_unlocked_item     [kernel.kallsyms]
             1707.00  1.0% __ticket_spin_trylock       [kernel.kallsyms]
             1688.00  1.0% xfs_perag_get_tag           [kernel.kallsyms]
             1660.00  1.0% flat_send_IPI_mask          [kernel.kallsyms]
             1538.00  0.9% xfs_inode_item_destroy      [kernel.kallsyms]
             1312.00  0.8% __shrink_dcache_sb          [kernel.kallsyms]
              940.00  0.6% xfs_perag_put               [kernel.kallsyms]

So there is some dentry cache reclaim going on.

FWIW, it appears there is quite a lot of RCU freeing overhead (~15% more CPU time) in the work kswapd is doing during these unlinks, too. I just had a look at kswapd when a 8-way create is running - it's running at 50-60% of a cpu for seconds at a time. I caught this while it was doing pure XFS inode cache reclaim (~10s sample, kswapd reclaimed ~1M inodes):

             samples  pcnt function                    DSO
             _______ _____ ___________________________ _________________

            27171.00  9.0% __call_rcu                  [kernel.kallsyms]
            21491.00  7.1% down_write                  [kernel.kallsyms]
            20916.00  6.9% xfs_reclaim_inode           [kernel.kallsyms]
            20313.00  6.7% radix_tree_delete           [kernel.kallsyms]
            15828.00  5.3% kmem_cache_free             [kernel.kallsyms]
            15819.00  5.2% xfs_idestroy_fork           [kernel.kallsyms]
            14893.00  4.9% is_bad_inode                [kernel.kallsyms]
            14666.00  4.9% _raw_spin_unlock_irqrestore [kernel.kallsyms]
            14191.00  4.7% xfs_reclaim_inode_grab      [kernel.kallsyms]
            14105.00  4.7% xfs_iunlock                 [kernel.kallsyms]
            10916.00  3.6% __ticket_spin_lock          [kernel.kallsyms]
            10125.00  3.4% xfs_iflush_cluster          [kernel.kallsyms]
             8221.00  2.7% xfs_qm_dqdetach             [kernel.kallsyms]
             7639.00  2.5% xfs_trans_unlocked_item     [kernel.kallsyms]
             7028.00  2.3% xfs_synchronize_times       [kernel.kallsyms]
             6974.00  2.3% up_write                    [kernel.kallsyms]
             5870.00  1.9% call_rcu_sched              [kernel.kallsyms]
             5634.00  1.9% _cond_resched               [kernel.kallsyms]

Which is showing a similar amount of RCU overhead as the unlink as above. And this while it was doing dentry cache reclaim (~10s sample):

            35921.00 15.7% __d_drop                      [kernel.kallsyms]
            30056.00 13.1% __ticket_spin_trylock         [kernel.kallsyms]
            29066.00 12.7% __ticket_spin_lock            [kernel.kallsyms]
            19043.00  8.3% __call_rcu                    [kernel.kallsyms]
            10098.00  4.4% iput                          [kernel.kallsyms]
             7013.00  3.1% __shrink_dcache_sb            [kernel.kallsyms]
             6774.00  3.0% __percpu_counter_add          [kernel.kallsyms]
             6708.00  2.9% radix_tree_tag_set            [kernel.kallsyms]
             5362.00  2.3% xfs_inactive                  [kernel.kallsyms]
             5130.00  2.2% __ticket_spin_unlock          [kernel.kallsyms]
             4884.00  2.1% call_rcu_sched                [kernel.kallsyms]
             4621.00  2.0% dentry_lru_del                [kernel.kallsyms]
             3735.00  1.6% bit_waitqueue                 [kernel.kallsyms]
             3727.00  1.6% dentry_iput                   [kernel.kallsyms]
             3473.00  1.5% shrink_icache_memory          [kernel.kallsyms]
             3279.00  1.4% kfree                         [kernel.kallsyms]
             3101.00  1.4% xfs_perag_get                 [kernel.kallsyms]
             2516.00  1.1% kmem_cache_free               [kernel.kallsyms]
             2272.00  1.0% shrink_dentry_list            [kernel.kallsyms]
Quoted 3 lines

I've never really seen any signficant dentry cache reclaim overhead in profiles of these workloads before, so this was a bit of a surprise....

call_rcu shouldn't be doing much, except for disabling irqs and linking the object into the list. I have a patch somewhere to reduce the irq disable overhead a bit, but it really shouldn't be doing a lot of work.

Sometimes you find that touching the rcu head field needs to get a cacheline exclusive, so a bit of work gets transferred there....

But it may also be something going a bit wrong in RCU. I blew it up once already, after the files_lock splitup that enabled all CPUs to create and destroy files :)

--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#95 lore raw
Quoted 181 lines · Nick Piggin
Quoted 73 lines · Dave Chinner
Quoted 72 lines · Dave Chinner
Quoted 65 lines · Nick Piggin
Quoted 55 lines · Dave Chinner
Quoted 11 lines · Nick Piggin

git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.git vfs-scale-working

Here is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:

* preparation patches
* introduce new locks to take over dcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking steps

Stress test doing:

       single thread 50M inode create
       single thread rm -rf
       2-way 50M inode create
       2-way rm -rf
       4-way 50M inode create
       4-way rm -rf
       8-way 50M inode create
       8-way rm -rf
       8-way 250M inode create
       8-way rm -rf

Failed about 5 minutes into the "4-way rm -rf" (~3 hours into the test) with a CPU stuck spinning on here:

[37372.084012] NMI backtrace for cpu 5
[37372.084012] CPU 5
[37372.084012] Modules linked in:
[37372.084012]
[37372.084012] Pid: 15214, comm: rm Not tainted 2.6.37-rc4-dgc+ #797 /Bochs
[37372.084012] RIP: 0010:[<ffffffff810643c4>]  [<ffffffff810643c4>] __ticket_spin_lock+0x14/0x20
[37372.084012] RSP: 0018:ffff880114643c98  EFLAGS: 00000213
[37372.084012] RAX: 0000000000008801 RBX: ffff8800687be6c0 RCX: ffff8800c4eb2688
[37372.084012] RDX: ffff880114643d38 RSI: ffff8800dfd4ea80 RDI: ffff880114643d14
[37372.084012] RBP: ffff880114643c98 R08: 0000000000000003 R09: 0000000000000000
[37372.084012] R10: 0000000000000000 R11: dead000000200200 R12: ffff880114643d14
[37372.084012] R13: ffff880114643cb8 R14: ffff880114643d38 R15: ffff8800687be71c
[37372.084012] FS:  00007fd6d7c93700(0000) GS:ffff8800dfd40000(0000) knlGS:0000000000000000
[37372.084012] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[37372.084012] CR2: 0000000000bbd108 CR3: 0000000107146000 CR4: 00000000000006e0
[37372.084012] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[37372.084012] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[37372.084012] Process rm (pid: 15214, threadinfo ffff880114642000, task ffff88011b16f890)
[37372.084012] Stack:
[37372.084012]  ffff880114643ca8 ffffffff81ad044e ffff880114643cf8 ffffffff81167ae7
[37372.084012]  0000000000000000 ffff880114643d38 000000000000000e ffff88011901d800
[37372.084012]  ffff8800cdb7cf5c ffff88011901d8e0 0000000000000000 0000000000000000
[37372.084012] Call Trace:
[37372.084012]  [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370
[37372.084012]  [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0
Quoted 8 lines
[37372.084012]  [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0
[37372.084012]  [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff8115daa2>] dentry_unhash+0x42/0x80
[37372.084012]  [<ffffffff8115db48>] vfs_rmdir+0x68/0x100
[37372.084012]  [<ffffffff8115fd93>] do_rmdir+0x113/0x130
[37372.084012]  [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90
[37372.084012]  [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40
[37372.084012]  [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
Quoted 3 lines

OK good, with any luck, that's the same bug.

Is this XFS?

Yes.

Quoted 7 lines
Quoted 7 lines
Quoted 1 line

Is there any concurrent activity happening on the same dentries?

Not from an application perspective.

Quoted 1 line

Ie. are the rm -rf threads running on the same directories,

No, each thread operating on a different directory.

This is probably fixed by the same patch as the lockdep splat trace.

Quoted 96 lines
Quoted 5 lines
Quoted 1 line

or is there any reclaim happening in the background?

IIRC, kswapd was consuming about 5-10% of a CPU during parallel unlink tests. Mainly reclaiming XFS inodes, I think, but there may be dentry cache reclaim going as well.

Turns out that the kswapd peaks are upwards of 50% of a CPU for a few seconds, then idle for 10-15s. Typical perf top output of kswapd while it is active during unlinks is:

             samples  pcnt function                    DSO
             _______ _____ ___________________________ _________________

            17168.00 10.2% __call_rcu                  [kernel.kallsyms]
            13223.00  7.8% kmem_cache_free             [kernel.kallsyms]
            12917.00  7.6% down_write                  [kernel.kallsyms]
            12665.00  7.5% xfs_iunlock                 [kernel.kallsyms]
            10493.00  6.2% xfs_reclaim_inode_grab      [kernel.kallsyms]
             9314.00  5.5% __lookup_tag                [kernel.kallsyms]
             9040.00  5.4% radix_tree_delete           [kernel.kallsyms]
             8694.00  5.1% is_bad_inode                [kernel.kallsyms]
             7639.00  4.5% __ticket_spin_lock          [kernel.kallsyms]
             6821.00  4.0% _raw_spin_unlock_irqrestore [kernel.kallsyms]
             5484.00  3.2% __d_drop                    [kernel.kallsyms]
             5114.00  3.0% xfs_reclaim_inode           [kernel.kallsyms]
             4626.00  2.7% __rcu_process_callbacks     [kernel.kallsyms]
             3556.00  2.1% up_write                    [kernel.kallsyms]
             3206.00  1.9% _cond_resched               [kernel.kallsyms]
             3129.00  1.9% xfs_qm_dqdetach             [kernel.kallsyms]
             2327.00  1.4% radix_tree_tag_clear        [kernel.kallsyms]
             2327.00  1.4% call_rcu_sched              [kernel.kallsyms]
             2262.00  1.3% __ticket_spin_unlock        [kernel.kallsyms]
             2215.00  1.3% xfs_ilock                   [kernel.kallsyms]
             2200.00  1.3% radix_tree_gang_lookup_tag  [kernel.kallsyms]
             1982.00  1.2% xfs_reclaim_inodes_ag       [kernel.kallsyms]
             1736.00  1.0% xfs_trans_unlocked_item     [kernel.kallsyms]
             1707.00  1.0% __ticket_spin_trylock       [kernel.kallsyms]
             1688.00  1.0% xfs_perag_get_tag           [kernel.kallsyms]
             1660.00  1.0% flat_send_IPI_mask          [kernel.kallsyms]
             1538.00  0.9% xfs_inode_item_destroy      [kernel.kallsyms]
             1312.00  0.8% __shrink_dcache_sb          [kernel.kallsyms]
              940.00  0.6% xfs_perag_put               [kernel.kallsyms]

So there is some dentry cache reclaim going on.

FWIW, it appears there is quite a lot of RCU freeing overhead (~15% more CPU time) in the work kswapd is doing during these unlinks, too. I just had a look at kswapd when a 8-way create is running - it's running at 50-60% of a cpu for seconds at a time. I caught this while it was doing pure XFS inode cache reclaim (~10s sample, kswapd reclaimed ~1M inodes):

             samples  pcnt function                    DSO
             _______ _____ ___________________________ _________________

            27171.00  9.0% __call_rcu                  [kernel.kallsyms]
            21491.00  7.1% down_write                  [kernel.kallsyms]
            20916.00  6.9% xfs_reclaim_inode           [kernel.kallsyms]
            20313.00  6.7% radix_tree_delete           [kernel.kallsyms]
            15828.00  5.3% kmem_cache_free             [kernel.kallsyms]
            15819.00  5.2% xfs_idestroy_fork           [kernel.kallsyms]
            14893.00  4.9% is_bad_inode                [kernel.kallsyms]
            14666.00  4.9% _raw_spin_unlock_irqrestore [kernel.kallsyms]
            14191.00  4.7% xfs_reclaim_inode_grab      [kernel.kallsyms]
            14105.00  4.7% xfs_iunlock                 [kernel.kallsyms]
            10916.00  3.6% __ticket_spin_lock          [kernel.kallsyms]
            10125.00  3.4% xfs_iflush_cluster          [kernel.kallsyms]
             8221.00  2.7% xfs_qm_dqdetach             [kernel.kallsyms]
             7639.00  2.5% xfs_trans_unlocked_item     [kernel.kallsyms]
             7028.00  2.3% xfs_synchronize_times       [kernel.kallsyms]
             6974.00  2.3% up_write                    [kernel.kallsyms]
             5870.00  1.9% call_rcu_sched              [kernel.kallsyms]
             5634.00  1.9% _cond_resched               [kernel.kallsyms]

Which is showing a similar amount of RCU overhead as the unlink as above. And this while it was doing dentry cache reclaim (~10s sample):

            35921.00 15.7% __d_drop                      [kernel.kallsyms]
            30056.00 13.1% __ticket_spin_trylock         [kernel.kallsyms]
            29066.00 12.7% __ticket_spin_lock            [kernel.kallsyms]
            19043.00  8.3% __call_rcu                    [kernel.kallsyms]
            10098.00  4.4% iput                          [kernel.kallsyms]
             7013.00  3.1% __shrink_dcache_sb            [kernel.kallsyms]
             6774.00  3.0% __percpu_counter_add          [kernel.kallsyms]
             6708.00  2.9% radix_tree_tag_set            [kernel.kallsyms]
             5362.00  2.3% xfs_inactive                  [kernel.kallsyms]
             5130.00  2.2% __ticket_spin_unlock          [kernel.kallsyms]
             4884.00  2.1% call_rcu_sched                [kernel.kallsyms]
             4621.00  2.0% dentry_lru_del                [kernel.kallsyms]
             3735.00  1.6% bit_waitqueue                 [kernel.kallsyms]
             3727.00  1.6% dentry_iput                   [kernel.kallsyms]
             3473.00  1.5% shrink_icache_memory          [kernel.kallsyms]
             3279.00  1.4% kfree                         [kernel.kallsyms]
             3101.00  1.4% xfs_perag_get                 [kernel.kallsyms]
             2516.00  1.1% kmem_cache_free               [kernel.kallsyms]
             2272.00  1.0% shrink_dentry_list            [kernel.kallsyms]
Quoted 7 lines
Quoted 3 lines

I've never really seen any signficant dentry cache reclaim overhead in profiles of these workloads before, so this was a bit of a surprise....

call_rcu shouldn't be doing much, except for disabling irqs and linking the object into the list. I have a patch somewhere to reduce the irq disable overhead a bit, but it really shouldn't be doing a lot of work.

Could you please enable CONFIG_RCU_TRACE, mount debugfs somewhere, and look at rcu/rcudata? There will be a "ql=" number printed for each CPU, and if that number is too large, __call_rcu() does take what it considers to be corrective action, which can incur some overhead.

If this is the problem, then increasing the value of the qhimark module parameter might help.

If this is not the problem, I could make a patch that disables some of __call_rcu()'s grace-period acceleration code if you are willing to try it out.

Quoted 6 lines

Sometimes you find that touching the rcu head field needs to get a cacheline exclusive, so a bit of work gets transferred there....

But it may also be something going a bit wrong in RCU. I blew it up once already, after the files_lock splitup that enabled all CPUs to create and destroy files :)

I would certainly like the opportunity to fix any bugs that might be in RCU... ;-)

Thanx, Paul

#96 lore raw
Quoted 195 lines · Paul E. McKenney
Quoted 181 lines · Nick Piggin
Quoted 73 lines · Dave Chinner
Quoted 72 lines · Dave Chinner
Quoted 65 lines · Nick Piggin
Quoted 55 lines · Dave Chinner
Quoted 11 lines · Nick Piggin

git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.git vfs-scale-working

Here is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:

* preparation patches
* introduce new locks to take over dcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking steps

Stress test doing:

       single thread 50M inode create
       single thread rm -rf
       2-way 50M inode create
       2-way rm -rf
       4-way 50M inode create
       4-way rm -rf
       8-way 50M inode create
       8-way rm -rf
       8-way 250M inode create
       8-way rm -rf

Failed about 5 minutes into the "4-way rm -rf" (~3 hours into the test) with a CPU stuck spinning on here:

[37372.084012] NMI backtrace for cpu 5
[37372.084012] CPU 5
[37372.084012] Modules linked in:
[37372.084012]
[37372.084012] Pid: 15214, comm: rm Not tainted 2.6.37-rc4-dgc+ #797 /Bochs
[37372.084012] RIP: 0010:[<ffffffff810643c4>]  [<ffffffff810643c4>] __ticket_spin_lock+0x14/0x20
[37372.084012] RSP: 0018:ffff880114643c98  EFLAGS: 00000213
[37372.084012] RAX: 0000000000008801 RBX: ffff8800687be6c0 RCX: ffff8800c4eb2688
[37372.084012] RDX: ffff880114643d38 RSI: ffff8800dfd4ea80 RDI: ffff880114643d14
[37372.084012] RBP: ffff880114643c98 R08: 0000000000000003 R09: 0000000000000000
[37372.084012] R10: 0000000000000000 R11: dead000000200200 R12: ffff880114643d14
[37372.084012] R13: ffff880114643cb8 R14: ffff880114643d38 R15: ffff8800687be71c
[37372.084012] FS:  00007fd6d7c93700(0000) GS:ffff8800dfd40000(0000) knlGS:0000000000000000
[37372.084012] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[37372.084012] CR2: 0000000000bbd108 CR3: 0000000107146000 CR4: 00000000000006e0
[37372.084012] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[37372.084012] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[37372.084012] Process rm (pid: 15214, threadinfo ffff880114642000, task ffff88011b16f890)
[37372.084012] Stack:
[37372.084012]  ffff880114643ca8 ffffffff81ad044e ffff880114643cf8 ffffffff81167ae7
[37372.084012]  0000000000000000 ffff880114643d38 000000000000000e ffff88011901d800
[37372.084012]  ffff8800cdb7cf5c ffff88011901d8e0 0000000000000000 0000000000000000
[37372.084012] Call Trace:
[37372.084012]  [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370
[37372.084012]  [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0
Quoted 8 lines
[37372.084012]  [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0
[37372.084012]  [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20
[37372.084012]  [<ffffffff8115daa2>] dentry_unhash+0x42/0x80
[37372.084012]  [<ffffffff8115db48>] vfs_rmdir+0x68/0x100
[37372.084012]  [<ffffffff8115fd93>] do_rmdir+0x113/0x130
[37372.084012]  [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90
[37372.084012]  [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40
[37372.084012]  [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
Quoted 3 lines

OK good, with any luck, that's the same bug.

Is this XFS?

Yes.

Quoted 7 lines
Quoted 7 lines
Quoted 1 line

Is there any concurrent activity happening on the same dentries?

Not from an application perspective.

Quoted 1 line

Ie. are the rm -rf threads running on the same directories,

No, each thread operating on a different directory.

This is probably fixed by the same patch as the lockdep splat trace.

Quoted 96 lines
Quoted 5 lines
Quoted 1 line

or is there any reclaim happening in the background?

IIRC, kswapd was consuming about 5-10% of a CPU during parallel unlink tests. Mainly reclaiming XFS inodes, I think, but there may be dentry cache reclaim going as well.

Turns out that the kswapd peaks are upwards of 50% of a CPU for a few seconds, then idle for 10-15s. Typical perf top output of kswapd while it is active during unlinks is:

             samples  pcnt function                    DSO
             _______ _____ ___________________________ _________________

            17168.00 10.2% __call_rcu                  [kernel.kallsyms]
            13223.00  7.8% kmem_cache_free             [kernel.kallsyms]
            12917.00  7.6% down_write                  [kernel.kallsyms]
            12665.00  7.5% xfs_iunlock                 [kernel.kallsyms]
            10493.00  6.2% xfs_reclaim_inode_grab      [kernel.kallsyms]
             9314.00  5.5% __lookup_tag                [kernel.kallsyms]
             9040.00  5.4% radix_tree_delete           [kernel.kallsyms]
             8694.00  5.1% is_bad_inode                [kernel.kallsyms]
             7639.00  4.5% __ticket_spin_lock          [kernel.kallsyms]
             6821.00  4.0% _raw_spin_unlock_irqrestore [kernel.kallsyms]
             5484.00  3.2% __d_drop                    [kernel.kallsyms]
             5114.00  3.0% xfs_reclaim_inode           [kernel.kallsyms]
             4626.00  2.7% __rcu_process_callbacks     [kernel.kallsyms]
             3556.00  2.1% up_write                    [kernel.kallsyms]
             3206.00  1.9% _cond_resched               [kernel.kallsyms]
             3129.00  1.9% xfs_qm_dqdetach             [kernel.kallsyms]
             2327.00  1.4% radix_tree_tag_clear        [kernel.kallsyms]
             2327.00  1.4% call_rcu_sched              [kernel.kallsyms]
             2262.00  1.3% __ticket_spin_unlock        [kernel.kallsyms]
             2215.00  1.3% xfs_ilock                   [kernel.kallsyms]
             2200.00  1.3% radix_tree_gang_lookup_tag  [kernel.kallsyms]
             1982.00  1.2% xfs_reclaim_inodes_ag       [kernel.kallsyms]
             1736.00  1.0% xfs_trans_unlocked_item     [kernel.kallsyms]
             1707.00  1.0% __ticket_spin_trylock       [kernel.kallsyms]
             1688.00  1.0% xfs_perag_get_tag           [kernel.kallsyms]
             1660.00  1.0% flat_send_IPI_mask          [kernel.kallsyms]
             1538.00  0.9% xfs_inode_item_destroy      [kernel.kallsyms]
             1312.00  0.8% __shrink_dcache_sb          [kernel.kallsyms]
              940.00  0.6% xfs_perag_put               [kernel.kallsyms]

So there is some dentry cache reclaim going on.

FWIW, it appears there is quite a lot of RCU freeing overhead (~15% more CPU time) in the work kswapd is doing during these unlinks, too. I just had a look at kswapd when a 8-way create is running - it's running at 50-60% of a cpu for seconds at a time. I caught this while it was doing pure XFS inode cache reclaim (~10s sample, kswapd reclaimed ~1M inodes):

             samples  pcnt function                    DSO
             _______ _____ ___________________________ _________________

            27171.00  9.0% __call_rcu                  [kernel.kallsyms]
            21491.00  7.1% down_write                  [kernel.kallsyms]
            20916.00  6.9% xfs_reclaim_inode           [kernel.kallsyms]
            20313.00  6.7% radix_tree_delete           [kernel.kallsyms]
            15828.00  5.3% kmem_cache_free             [kernel.kallsyms]
            15819.00  5.2% xfs_idestroy_fork           [kernel.kallsyms]
            14893.00  4.9% is_bad_inode                [kernel.kallsyms]
            14666.00  4.9% _raw_spin_unlock_irqrestore [kernel.kallsyms]
            14191.00  4.7% xfs_reclaim_inode_grab      [kernel.kallsyms]
            14105.00  4.7% xfs_iunlock                 [kernel.kallsyms]
            10916.00  3.6% __ticket_spin_lock          [kernel.kallsyms]
            10125.00  3.4% xfs_iflush_cluster          [kernel.kallsyms]
             8221.00  2.7% xfs_qm_dqdetach             [kernel.kallsyms]
             7639.00  2.5% xfs_trans_unlocked_item     [kernel.kallsyms]
             7028.00  2.3% xfs_synchronize_times       [kernel.kallsyms]
             6974.00  2.3% up_write                    [kernel.kallsyms]
             5870.00  1.9% call_rcu_sched              [kernel.kallsyms]
             5634.00  1.9% _cond_resched               [kernel.kallsyms]

Which is showing a similar amount of RCU overhead as the unlink as above. And this while it was doing dentry cache reclaim (~10s sample):

            35921.00 15.7% __d_drop                      [kernel.kallsyms]
            30056.00 13.1% __ticket_spin_trylock         [kernel.kallsyms]
            29066.00 12.7% __ticket_spin_lock            [kernel.kallsyms]
            19043.00  8.3% __call_rcu                    [kernel.kallsyms]
            10098.00  4.4% iput                          [kernel.kallsyms]
             7013.00  3.1% __shrink_dcache_sb            [kernel.kallsyms]
             6774.00  3.0% __percpu_counter_add          [kernel.kallsyms]
             6708.00  2.9% radix_tree_tag_set            [kernel.kallsyms]
             5362.00  2.3% xfs_inactive                  [kernel.kallsyms]
             5130.00  2.2% __ticket_spin_unlock          [kernel.kallsyms]
             4884.00  2.1% call_rcu_sched                [kernel.kallsyms]
             4621.00  2.0% dentry_lru_del                [kernel.kallsyms]
             3735.00  1.6% bit_waitqueue                 [kernel.kallsyms]
             3727.00  1.6% dentry_iput                   [kernel.kallsyms]
             3473.00  1.5% shrink_icache_memory          [kernel.kallsyms]
             3279.00  1.4% kfree                         [kernel.kallsyms]
             3101.00  1.4% xfs_perag_get                 [kernel.kallsyms]
             2516.00  1.1% kmem_cache_free               [kernel.kallsyms]
             2272.00  1.0% shrink_dentry_list            [kernel.kallsyms]
Quoted 7 lines
Quoted 3 lines

I've never really seen any signficant dentry cache reclaim overhead in profiles of these workloads before, so this was a bit of a surprise....

call_rcu shouldn't be doing much, except for disabling irqs and linking the object into the list. I have a patch somewhere to reduce the irq disable overhead a bit, but it really shouldn't be doing a lot of work.

Could you please enable CONFIG_RCU_TRACE, mount debugfs somewhere, and look at rcu/rcudata? There will be a "ql=" number printed for each CPU, and if that number is too large, __call_rcu() does take what it considers to be corrective action, which can incur some overhead.

Quoted 6 lines

If this is the problem, then increasing the value of the qhimark module parameter might help.

If this is not the problem, I could make a patch that disables some of __call_rcu()'s grace-period acceleration code if you are willing to try it out.

Another thing that might help is to reduce the value of CONFIG_RCU_FANOUT to something like 16. If this does help, then there is a reasonably straightforward change I can make to RCU.

Thanx, Paul

Quoted 11 lines
Quoted 6 lines

Sometimes you find that touching the rcu head field needs to get a cacheline exclusive, so a bit of work gets transferred there....

But it may also be something going a bit wrong in RCU. I blew it up once already, after the files_lock splitup that enabled all CPUs to create and destroy files :)

I would certainly like the opportunity to fix any bugs that might be in RCU... ;-)

Thanx, Paul

--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html