rcu-walk and dcache scaling
Subsystems (58)
PATCH 39/46 fs: d_revalidate_rcu for rcu-walk
In reply to Nick PigginThis seems to be the best way to get filesystems aware of RCU walking in their d_revalidate routines. Needs more documentation if it works out OK.
--- fs/dcache.c | 2 + fs/ecryptfs/dentry.c | 15 ++++++++---- fs/fat/namei_vfat.c | 14 ++++++------ fs/fuse/dir.c | 7 ++++- fs/namei.c | 56 ++++++++++++++++++++++++++++++++++------------- fs/sysfs/dir.c | 18 ++++++++++++-- include/linux/dcache.h | 7 +++++- 7 files changed, 85 insertions(+), 34 deletions(-)
fs/dcache.c +2 -0
@@ -1243,6 +1243,8 @@ void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op) dentry->d_flags |=DCACHE_OP_COMPARE ;if (op->d_revalidate) dentry->d_flags |=DCACHE_OP_REVALIDATE ; +if (op->d_revalidate_rcu) + dentry->d_flags |=DCACHE_OP_REVALIDATE_RCU ;if (op->d_delete) dentry->d_flags |=DCACHE_OP_DELETE ;
fs/ecryptfs/dentry.c +10 -5
@@ -30,7 +30,7 @@#include "ecryptfs_kernel.h" /** -* ecryptfs_d_revalidate - revalidate an ecryptfs dentry +* ecryptfs_d_revalidate_rcu - revalidate an ecryptfs dentry * @dentry: The ecryptfs dentry * @nd: The associated nameidata *
@@ -42,7 +42,7 @@* Returns 1 if valid, 0 otherwise. * */ -static int ecryptfs_d_revalidate (struct dentry *dentry,struct nameidata *nd) +static int ecryptfs_d_revalidate_rcu (struct dentry *dentry,struct nameidata *nd) {struct dentry *lower_dentry =ecryptfs_dentry_to_lower (dentry);struct vfsmount *lower_mnt =ecryptfs_dentry_to_lower_mnt (dentry);
@@ -50,13 +50,18 @@ static int ecryptfs_d_revalidate(struct dentry *dentry, struct nameidata *nd)struct vfsmount *vfsmount_save;int rc =1 ; -if (!lower_dentry->d_op || !lower_dentry->d_op->d_revalidate) +if (!(lower_dentry->d_flags &DCACHE_OP_REVALIDATE_EITHER )) goto out; +if (nd->flags &LOOKUP_RCU ) +return -ECHILD ; dentry_save = nd->path.dentry; vfsmount_save = nd->path.mnt; nd->path.dentry = lower_dentry; nd->path.mnt = lower_mnt; - rc = lower_dentry->d_op->d_revalidate (lower_dentry, nd); +if (lower_dentry->d_flags &DCACHE_OP_REVALIDATE ) + rc = lower_dentry->d_op->d_revalidate (lower_dentry, nd); +else + rc = lower_dentry->d_op->d_revalidate_rcu (lower_dentry, nd); nd->path.dentry = dentry_save; nd->path.mnt = vfsmount_save;if (dentry->d_inode) {
@@ -91,6 +96,6 @@ static void ecryptfs_d_release(struct dentry *dentry) }const struct dentry_operations ecryptfs_dops = { - .d_revalidate = ecryptfs_d_revalidate, + .d_revalidate_rcu = ecryptfs_d_revalidate_rcu, .d_release = ecryptfs_d_release, };
fs/fat/namei_vfat.c +7 -7
@@ -31,7 +31,7 @@* If it happened, the negative dentry isn't actually negative * anymore. So, drop it. */ -static int vfat_revalidate_shortname (struct dentry *dentry) +static int vfat_revalidate_rcu_shortname (struct dentry *dentry) {int ret =1 ;spin_lock (&dentry->d_lock);
@@ -41,15 +41,15 @@ static int vfat_revalidate_shortname(struct dentry *dentry)return ret; } -static int vfat_revalidate (struct dentry *dentry,struct nameidata *nd) +static int vfat_revalidate_rcu (struct dentry *dentry,struct nameidata *nd) {/* This is not negative dentry. Always valid. */ if (dentry->d_inode)return 1 ; -return vfat_revalidate_shortname (dentry); +return vfat_revalidate_rcu_shortname (dentry); } -static int vfat_revalidate_ci (struct dentry *dentry,struct nameidata *nd) +static int vfat_revalidate_rcu_ci (struct dentry *dentry,struct nameidata *nd) {/* * This is not negative dentry. Always valid.
@@ -81,7 +81,7 @@ static int vfat_revalidate_ci(struct dentry *dentry, struct nameidata *nd)return 0 ; } -return vfat_revalidate_shortname (dentry); +return vfat_revalidate_rcu_shortname (dentry); }/* returns the length of a struct qstr, ignoring trailing dots */
@@ -175,13 +175,13 @@ static int vfat_cmp(const struct dentry *parent, }static const struct dentry_operations vfat_ci_dentry_ops = { - .d_revalidate = vfat_revalidate_ci, + .d_revalidate = vfat_revalidate_rcu_ci, .d_hash = vfat_hashi, .d_compare = vfat_cmpi, };static const struct dentry_operations vfat_dentry_ops = { - .d_revalidate = vfat_revalidate, + .d_revalidate = vfat_revalidate_rcu, .d_hash = vfat_hash, .d_compare = vfat_cmp, };
fs/fuse/dir.c +5 -2
@@ -154,7 +154,7 @@ u64 fuse_get_attr_version(struct fuse_conn *fc)* the lookup once more. If the lookup results in the same inode, * then refresh the attributes, timeouts and mark the dentry valid. */ -static int fuse_dentry_revalidate (struct dentry *entry,struct nameidata *nd) +static int fuse_dentry_revalidate_rcu (struct dentry *entry,struct nameidata *nd) {struct inode *inode = entry->d_inode;
@@ -169,6 +169,9 @@ static int fuse_dentry_revalidate(struct dentry *entry, struct nameidata *nd)struct dentry *parent;u64 attr_version; +if (nd->flags &LOOKUP_RCU ) +return -ECHILD ; +/* For negative dentries, always do a fresh lookup */ if (!inode)return 0 ;
@@ -225,7 +228,7 @@ static int invalid_nodeid(u64 nodeid) }const struct dentry_operations fuse_dentry_operations = { - .d_revalidate = fuse_dentry_revalidate, + .d_revalidate_rcu = fuse_dentry_revalidate_rcu, };int fuse_valid_type (int m)
fs/namei.c +40 -16
111 lines not on this page · raw message
fs/sysfs/dir.c +15 -3
@@ -237,12 +237,21 @@ static int sysfs_dentry_delete(const struct dentry *dentry)return !!(sd->s_flags &SYSFS_FLAG_REMOVED ); } -static int sysfs_dentry_revalidate (struct dentry *dentry,struct nameidata *nd) +static int sysfs_dentry_revalidate_rcu (struct dentry *dentry,struct nameidata *nd) {struct sysfs_dirent *sd = dentry->d_fsdata;int is_dir; -mutex_lock (&sysfs_mutex); +if (nd->flags &LOOKUP_RCU ) { +if (!mutex_trylock (&sysfs_mutex)) +return -ECHILD ; +/* ensure dentry still exists, now under the sysfs_mutex */ +if (read_seqcount_retry (&dentry->d_seq, nd->seq)) { +mutex_unlock (&sysfs_mutex); +return -ECHILD ; + } + }else +mutex_lock (&sysfs_mutex);/* The sysfs dirent has been deleted */ if (sd->s_flags &SYSFS_FLAG_REMOVED )
@@ -272,6 +281,9 @@ out_bad:*/ is_dir = (sysfs_type (sd) ==SYSFS_DIR );mutex_unlock (&sysfs_mutex); +if (nd->flags &LOOKUP_RCU ) +return -ECHILD ; +if (is_dir) {/* If we have submounts we must allow the vfs caches * to lie about the state of the filesystem to prevent
@@ -294,7 +306,7 @@ static void sysfs_dentry_iput(struct dentry *dentry, struct inode *inode) }static const struct dentry_operations sysfs_dentry_ops = { - .d_revalidate = sysfs_dentry_revalidate, + .d_revalidate_rcu = sysfs_dentry_revalidate_rcu, .d_delete = sysfs_dentry_delete, .d_iput = sysfs_dentry_iput, };
include/linux/dcache.h +6 -1
@@ -132,6 +132,7 @@ enum dentry_d_lock_classstruct dentry_operations {int (*d_revalidate)(struct dentry *,struct nameidata *); +int (*d_revalidate_rcu)(struct dentry *,struct nameidata *);int (*d_hash)(const struct dentry *,const struct inode *,struct qstr *); int (*d_compare)(const struct dentry *,
@@ -184,8 +185,12 @@ struct dentry_operations {#define DCACHE_OP_HASH 0x1000#define DCACHE_OP_COMPARE 0x2000#define DCACHE_OP_REVALIDATE 0x4000 -#define DCACHE_OP_DELETE 0x8000 +#define DCACHE_OP_REVALIDATE_RCU 0x8000 +#define DCACHE_OP_DELETE 0x10000 + +#define DCACHE_OP_REVALIDATE_EITHER \ + (DCACHE_OP_REVALIDATE|DCACHE_OP_REVALIDATE_RCU)extern spinlock_t dcache_inode_lock;extern spinlock_t dcache_hash_lock;
Signature · 1 line
1.7.1
PATCH 40/46 fs: provide rcu-walk aware permission i_ops
In reply to Nick Piggin--- fs/namei.c | 131 ++++++++++++++++++++++++++++++++++++++++------------ include/linux/fs.h | 6 ++ 2 files changed, 107 insertions(+), 30 deletions(-)
fs/namei.c +101 -30
@@ -169,8 +169,36 @@ EXPORT_SYMBOL(putname);/* * This does basic POSIX ACL permission checking */ -static inline int __acl_permission_check (struct inode *inode,int mask, -int (*check_acl)(struct inode *inode,int mask),int rcu) +static int acl_permission_check_rcu (struct inode *inode,int mask,unsigned int flags, +int (*check_acl_rcu)(struct inode *inode,int mask,unsigned int flags)) +{ +umode_t mode = inode->i_mode; + + mask &=MAY_READ |MAY_WRITE |MAY_EXEC ; + +if (current_fsuid () == inode->i_uid) + mode >>=6 ; +else { +if (IS_POSIXACL (inode) && (mode &S_IRWXG ) && check_acl_rcu) { +int error =check_acl_rcu (inode, mask, flags); +if (error != -EAGAIN ) +return error; + } + +if (in_group_p (inode->i_gid)) + mode >>=3 ; + } + +/* +* If the DACs are ok we don't need any capability check. +*/ +if ((mask & ~mode) ==0 ) +return 0 ; +return -EACCES ; +} + +static int acl_permission_check (struct inode *inode,int mask,unsigned int flags, +int (*check_acl)(struct inode *inode,int mask)) {umode_t mode = inode->i_mode;
@@ -180,7 +208,7 @@ static inline int __acl_permission_check(struct inode *inode, int mask, mode >>=6 ; else {if (IS_POSIXACL (inode) && (mode &S_IRWXG ) && check_acl) { -if (rcu) { +if (flags) {return -ECHILD ; }else {int error =check_acl (inode, mask);
@@ -201,10 +229,52 @@ static inline int __acl_permission_check(struct inode *inode, int mask,return -EACCES ; } -static inline int acl_permission_check (struct inode *inode,int mask, -int (*check_acl)(struct inode *inode,int mask)) +/** +* generic_permission_rcu - check for access rights on a Posix-like filesystem +* @inode: inode to check access rights for +* @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC) +* @check_acl_rcu: optional callback to check for Posix ACLs +* @flags IPERM_FLAG_ flags. +* +* Used to check for read/write/execute permissions on a file. +* We use "fsuid" for this, letting us set arbitrary permissions +* for filesystem access without changing the "normal" uids which +* are used for other things. +* +* generic_permission_rcu must be rcu-walk aware. It should return +* -ECHILD in case an rcu-walk request cannot be satisfied (eg. +* requires blocking or too much thought!). It would then be called +* again in ref-walk mode. +*/ +int generic_permission_rcu (struct inode *inode,int mask,unsigned int flags, +int (*check_acl_rcu)(struct inode *inode,int mask,unsigned int flags)) { -return __acl_permission_check (inode, mask, check_acl,0 ); +int ret; + +/* +* Do the basic POSIX ACL permission checks. +*/ + ret =acl_permission_check_rcu (inode, mask, flags, check_acl_rcu); +if (ret != -EACCES ) +return ret; + +/* +* Read/write DACs are always overridable. +* Executable DACs are overridable if at least one exec bit is set. +*/ +if (!(mask &MAY_EXEC ) ||execute_ok (inode)) +if (capable (CAP_DAC_OVERRIDE )) +return 0 ; + +/* +* Searching includes executable on directories, else just read. +*/ + mask &=MAY_READ |MAY_WRITE |MAY_EXEC ; +if (mask ==MAY_READ || (S_ISDIR (inode->i_mode) && !(mask &MAY_WRITE ))) +if (capable (CAP_DAC_READ_SEARCH )) +return 0 ; + +return -EACCES ; }/**
@@ -226,7 +296,7 @@ int generic_permission(struct inode *inode, int mask,/* * Do the basic POSIX ACL permission checks. */ - ret =acl_permission_check (inode, mask, check_acl); + ret =acl_permission_check (inode, mask,0 , check_acl);if (ret != -EACCES )return ret;
@@ -282,8 +352,14 @@ int inode_permission(struct inode *inode, int mask)if (inode->i_op->permission) retval = inode->i_op->permission (inode, mask); +else if (inode->i_op->permission_rcu) + retval = inode->i_op->permission_rcu (inode, mask,0 ); +else if (inode->i_op->check_acl_rcu) + retval =generic_permission_rcu (inode, mask,0 , + inode->i_op->check_acl_rcu);else - retval =generic_permission (inode, mask, inode->i_op->check_acl); + retval =generic_permission (inode, mask, + inode->i_op->check_acl);if (retval)return retval;
@@ -622,22 +698,26 @@ force_reval_path(struct path *path, struct nameidata *nd)* short-cut DAC fails, then call ->permission() to do more * complete permission check. */ -static inline int __exec_permission (struct inode *inode,int rcu) +static inline int exec_permission (struct inode *inode,unsigned int flags) {int ret; -if (inode->i_op->permission) { -if (rcu) +if (inode->i_op->permission_rcu) { + ret = inode->i_op->permission_rcu (inode,MAY_EXEC , flags); + }else if (inode->i_op->permission) { +if (flags)return -ECHILD ; ret = inode->i_op->permission (inode,MAY_EXEC ); -if (!ret) - goto ok; -return ret; + }else if (inode->i_op->check_acl_rcu) { + ret =acl_permission_check_rcu (inode,MAY_EXEC , flags, + inode->i_op->check_acl_rcu); + }else { + ret =acl_permission_check (inode,MAY_EXEC , flags, + inode->i_op->check_acl); } - ret =__acl_permission_check (inode,MAY_EXEC , inode->i_op->check_acl, rcu); -if (!ret) +if (likely (!ret)) goto ok; -if (rcu && ret == -ECHILD ) +if (ret == -ECHILD )return ret;if (capable (CAP_DAC_OVERRIDE ) ||capable (CAP_DAC_READ_SEARCH ))
@@ -648,16 +728,6 @@ ok:return security_inode_permission (inode,MAY_EXEC );/* XXX: ok for RCU? */ } -static int exec_permission (struct inode *inode) -{ -return __exec_permission (inode,0 ); -} - -static int exec_permission_rcu (struct inode *inode) -{ -return __exec_permission (inode,1 ); -} - static __always_inline void set_root(struct nameidata *nd) {if (!nd->root.mnt)
@@ -1126,7 +1196,7 @@ static int link_path_walk(const char *name, struct nameidata *nd) nd->flags |=LOOKUP_CONTINUE ;if (nd->flags &LOOKUP_RCU ) { - err =exec_permission_rcu (nd->inode); + err =exec_permission (nd->inode,IPERM_FLAG_RCU );if (err == -ECHILD ) {if (nameidata_drop_rcu (nd))return -ECHILD ;
@@ -1134,7 +1204,7 @@ static int link_path_walk(const char *name, struct nameidata *nd) } }else { exec_again: - err =exec_permission (nd->inode); + err =exec_permission (nd->inode,0 ); }if (err)break ;
@@ -1567,7 +1637,7 @@ static struct dentry *__lookup_hash(struct qstr *name,struct dentry *dentry;int err; - err =exec_permission (inode); + err =exec_permission (inode,0 );if (err)return ERR_PTR (err);
@@ -3352,6 +3422,7 @@ EXPORT_SYMBOL(vfs_follow_link);EXPORT_SYMBOL (vfs_link);EXPORT_SYMBOL (vfs_mkdir);EXPORT_SYMBOL (vfs_mknod); +EXPORT_SYMBOL (generic_permission_rcu);EXPORT_SYMBOL (generic_permission);EXPORT_SYMBOL (vfs_readlink);EXPORT_SYMBOL (vfs_rename);
include/linux/fs.h +6 -0
@@ -1549,11 +1549,15 @@ struct file_operations {int (*setlease)(struct file *,long ,struct file_lock **); }; +#define IPERM_FLAG_RCU 0x0001 +struct inode_operations {struct dentry * (*lookup) (struct inode *,struct dentry *,struct nameidata *);void * (*follow_link) (struct dentry *,struct nameidata *);int (*permission) (struct inode *,int ); +int (*permission_rcu) (struct inode *,int ,unsigned int );int (*check_acl)(struct inode *,int ); +int (*check_acl_rcu)(struct inode *,int ,unsigned int );int (*readlink) (struct dentry *,char __user *,int );void (*put_link) (struct dentry *,struct nameidata *,void *);
@@ -2164,6 +2168,8 @@ extern sector_t bmap(struct inode *, sector_t);#endif extern int notify_change (struct dentry *,struct iattr *);extern int inode_permission (struct inode *,int ); +extern int generic_permission_rcu (struct inode *,int ,unsigned int , +int (*check_acl_rcu)(struct inode *,int ,unsigned int ));extern int generic_permission (struct inode *,int ,int (*check_acl)(struct inode *,int ));
Signature · 1 line
1.7.1
PATCH 41/46 fs: provide simple rcu-walk ACL implementation
In reply to Nick PigginThis simple implementation just checks for no ACLs on the inode, and if so, then the rcu-walk may proceed, otherwise fail it.
This could easily be extended to put acls under RCU and check them under seqlock, if need be. But this implementation is enough to show the rcu-walk aware permissions code for path lookups is working, and will handle cases where there are no ACLs or ACLs in just the final element.
Convert tmpfs, ext2, btrfs. Each of these uses acl/permission code in a different way, so convert them all to provide templates and proof of concept.
--- fs/btrfs/acl.c | 19 ++++++++++++------- fs/btrfs/ctree.h | 4 ++-- fs/btrfs/inode.c | 14 +++++++------- fs/ext2/acl.c | 11 +++++++++-- fs/ext2/acl.h | 8 ++++---- fs/ext2/file.c | 2 +- fs/ext2/namei.c | 4 ++-- fs/ext3/acl.c | 11 +++++++++-- fs/ext3/acl.h | 8 ++++---- fs/ext3/file.c | 2 +- fs/ext3/namei.c | 4 ++-- fs/ext4/acl.c | 11 +++++++++-- fs/ext4/acl.h | 4 ++-- fs/ext4/file.c | 2 +- fs/ext4/namei.c | 4 ++-- fs/generic_acl.c | 20 ++++++++++++++++++++ fs/xfs/linux-2.6/xfs_acl.c | 8 +++++++- fs/xfs/linux-2.6/xfs_iops.c | 8 ++++---- fs/xfs/xfs_acl.h | 4 ++-- include/linux/generic_acl.h | 1 + include/linux/posix_acl.h | 19 +++++++++++++++++++ mm/shmem.c | 6 +++--- 22 files changed, 123 insertions(+), 51 deletions(-)
fs/btrfs/acl.c +12 -7
@@ -185,18 +185,23 @@ static int btrfs_xattr_acl_set(struct dentry *dentry, const char *name,return ret; } -int btrfs_check_acl (struct inode *inode,int mask) +int btrfs_check_acl_rcu (struct inode *inode,int mask,unsigned int flags) {struct posix_acl *acl;int error = -EAGAIN ; - acl =btrfs_get_acl (inode,ACL_TYPE_ACCESS ); +if (flags &IPERM_FLAG_RCU ) { +if (!negative_cached_acl (inode,ACL_TYPE_ACCESS )) +return -ECHILD ; + }else { + acl =btrfs_get_acl (inode,ACL_TYPE_ACCESS ); -if (IS_ERR (acl)) -return PTR_ERR (acl); -if (acl) { - error =posix_acl_permission (inode, acl, mask); -posix_acl_release (acl); +if (IS_ERR (acl)) +return PTR_ERR (acl); +if (acl) { + error =posix_acl_permission (inode, acl, mask); +posix_acl_release (acl); + } }return error;
fs/btrfs/ctree.h +2 -2
@@ -2544,9 +2544,9 @@ int btrfs_sync_fs(struct super_block *sb, int wait);/* acl.c */ #ifdef CONFIG_BTRFS_FS_POSIX_ACL -int btrfs_check_acl (struct inode *inode,int mask); +int btrfs_check_acl_rcu (struct inode *inode,int mask,unsigned int flags);#else -#define btrfs_check_acl NULL +#define btrfs_check_acl_rcu NULL#endif int btrfs_init_acl (struct btrfs_trans_handle *trans,struct inode *inode,struct inode *dir);
fs/btrfs/inode.c +7 -7
@@ -7033,11 +7033,11 @@ static int btrfs_set_page_dirty(struct page *page)return __set_page_dirty_nobuffers (page); } -static int btrfs_permission (struct inode *inode,int mask) +static int btrfs_permission_rcu (struct inode *inode,int mask,unsigned int flags) {if ((BTRFS_I (inode)->flags &BTRFS_INODE_READONLY ) && (mask &MAY_WRITE ))return -EACCES ; -return generic_permission (inode, mask, btrfs_check_acl); +return generic_permission_rcu (inode, mask, flags, btrfs_check_acl_rcu); }static const struct inode_operations btrfs_dir_inode_operations = {
@@ -7056,11 +7056,11 @@ static const struct inode_operations btrfs_dir_inode_operations = { .getxattr = btrfs_getxattr, .listxattr = btrfs_listxattr, .removexattr = btrfs_removexattr, - .permission = btrfs_permission, + .permission_rcu = btrfs_permission_rcu, };static const struct inode_operations btrfs_dir_ro_inode_operations = { .lookup = btrfs_lookup, - .permission = btrfs_permission, + .permission_rcu = btrfs_permission_rcu, };static const struct file_operations btrfs_dir_file_operations = {
@@ -7129,14 +7129,14 @@ static const struct inode_operations btrfs_file_inode_operations = { .getxattr = btrfs_getxattr, .listxattr = btrfs_listxattr, .removexattr = btrfs_removexattr, - .permission = btrfs_permission, + .permission_rcu = btrfs_permission_rcu, .fallocate = btrfs_fallocate, .fiemap = btrfs_fiemap, };static const struct inode_operations btrfs_special_inode_operations = { .getattr = btrfs_getattr, .setattr = btrfs_setattr, - .permission = btrfs_permission, + .permission_rcu = btrfs_permission_rcu, .setxattr = btrfs_setxattr, .getxattr = btrfs_getxattr, .listxattr = btrfs_listxattr,
@@ -7146,7 +7146,7 @@ static const struct inode_operations btrfs_symlink_inode_operations = { .readlink = generic_readlink, .follow_link = page_follow_link_light, .put_link = page_put_link, - .permission = btrfs_permission, + .permission_rcu = btrfs_permission_rcu, .setxattr = btrfs_setxattr, .getxattr = btrfs_getxattr, .listxattr = btrfs_listxattr,
fs/ext2/acl.c +9 -2
@@ -232,10 +232,17 @@ ext2_set_acl(struct inode *inode, int type, struct posix_acl *acl) }int -ext2_check_acl (struct inode *inode,int mask) +ext2_check_acl_rcu (struct inode *inode,int mask,unsigned int flags) { -struct posix_acl *acl =ext2_get_acl (inode,ACL_TYPE_ACCESS ); +struct posix_acl *acl; + +if (flags &IPERM_FLAG_RCU ) { +if (!negative_cached_acl (inode,ACL_TYPE_ACCESS )) +return -ECHILD ; +return -EAGAIN ; + } + acl =ext2_get_acl (inode,ACL_TYPE_ACCESS );if (IS_ERR (acl))return PTR_ERR (acl);if (acl) {
fs/ext2/acl.h +4 -4
@@ -54,15 +54,15 @@ static inline int ext2_acl_count(size_t size)#ifdef CONFIG_EXT2_FS_POSIX_ACL /* acl.c */ -extern int ext2_check_acl (struct inode *,int ); +extern int ext2_check_acl_rcu (struct inode *,int ,unsigned int );extern int ext2_acl_chmod (struct inode *);extern int ext2_init_acl (struct inode *,struct inode *);#else #include <linux/sched.h> -#define ext2_check_acl NULL -#define ext2_get_acl NULL -#define ext2_set_acl NULL +#define ext2_check_acl_rcu NULL +#define ext2_get_acl NULL +#define ext2_set_acl NULLstatic inline int ext2_acl_chmod (struct inode *inode)
fs/ext2/file.c +1 -1
@@ -102,6 +102,6 @@ const struct inode_operations ext2_file_inode_operations = { .removexattr = generic_removexattr, #endif .setattr = ext2_setattr, - .check_acl = ext2_check_acl, + .check_acl_rcu = ext2_check_acl_rcu, .fiemap = ext2_fiemap, };
fs/ext2/namei.c +2 -2
@@ -417,7 +417,7 @@ const struct inode_operations ext2_dir_inode_operations = { .removexattr = generic_removexattr, #endif .setattr = ext2_setattr, - .check_acl = ext2_check_acl, + .check_acl_rcu = ext2_check_acl_rcu, }; const struct inode_operations ext2_special_inode_operations = {
@@ -428,5 +428,5 @@ const struct inode_operations ext2_special_inode_operations = { .removexattr = generic_removexattr, #endif .setattr = ext2_setattr, - .check_acl = ext2_check_acl, + .check_acl_rcu = ext2_check_acl_rcu, };
fs/ext3/acl.c +9 -2
@@ -240,10 +240,17 @@ ext3_set_acl(handle_t *handle, struct inode *inode, int type, }int -ext3_check_acl (struct inode *inode,int mask) +ext3_check_acl_rcu (struct inode *inode,int mask,unsigned int flags) { -struct posix_acl *acl =ext3_get_acl (inode,ACL_TYPE_ACCESS ); +struct posix_acl *acl; + +if (flags &IPERM_FLAG_RCU ) { +if (!negative_cached_acl (inode,ACL_TYPE_ACCESS )) +return -ECHILD ; +return -EAGAIN ; + } + acl =ext3_get_acl (inode,ACL_TYPE_ACCESS );if (IS_ERR (acl))return PTR_ERR (acl);if (acl) {
fs/ext3/acl.h +4 -4
@@ -54,13 +54,13 @@ static inline int ext3_acl_count(size_t size)#ifdef CONFIG_EXT3_FS_POSIX_ACL /* acl.c */ -extern int ext3_check_acl (struct inode *,int ); -extern int ext3_acl_chmod (struct inode *); -extern int ext3_init_acl (handle_t *,struct inode *,struct inode *); +extern int ext3_check_acl_rcu (struct inode *,int ,unsigned int ); +extern int ext3_acl_chmod (struct inode *); +extern int ext3_init_acl (handle_t *,struct inode *,struct inode *);#else /* CONFIG_EXT3_FS_POSIX_ACL */ #include <linux/sched.h> -#define ext3_check_acl NULL +#define ext3_check_acl_rcu NULLstatic inline int ext3_acl_chmod (struct inode *inode)
fs/ext3/file.c +1 -1
@@ -79,7 +79,7 @@ const struct inode_operations ext3_file_inode_operations = { .listxattr = ext3_listxattr, .removexattr = generic_removexattr, #endif - .check_acl = ext3_check_acl, + .check_acl_rcu = ext3_check_acl_rcu, .fiemap = ext3_fiemap, };
fs/ext3/namei.c +2 -2
@@ -2464,7 +2464,7 @@ const struct inode_operations ext3_dir_inode_operations = { .listxattr = ext3_listxattr, .removexattr = generic_removexattr, #endif - .check_acl = ext3_check_acl, + .check_acl_rcu = ext3_check_acl_rcu, }; const struct inode_operations ext3_special_inode_operations = {
@@ -2475,5 +2475,5 @@ const struct inode_operations ext3_special_inode_operations = { .listxattr = ext3_listxattr, .removexattr = generic_removexattr, #endif - .check_acl = ext3_check_acl, + .check_acl_rcu = ext3_check_acl_rcu, };
fs/ext4/acl.c +9 -2
@@ -238,10 +238,17 @@ ext4_set_acl(handle_t *handle, struct inode *inode, int type, }int -ext4_check_acl (struct inode *inode,int mask) +ext4_check_acl_rcu (struct inode *inode,int mask,unsigned int flags) { -struct posix_acl *acl =ext4_get_acl (inode,ACL_TYPE_ACCESS ); +struct posix_acl *acl; + +if (flags &IPERM_FLAG_RCU ) { +if (!negative_cached_acl (inode,ACL_TYPE_ACCESS )) +return -ECHILD ; +return -EAGAIN ; + } + acl =ext4_get_acl (inode,ACL_TYPE_ACCESS );if (IS_ERR (acl))return PTR_ERR (acl);if (acl) {
fs/ext4/acl.h +2 -2
@@ -54,13 +54,13 @@ static inline int ext4_acl_count(size_t size)#ifdef CONFIG_EXT4_FS_POSIX_ACL /* acl.c */ -extern int ext4_check_acl (struct inode *,int ); +extern int ext4_check_acl_rcu (struct inode *,int ,unsigned int );extern int ext4_acl_chmod (struct inode *);extern int ext4_init_acl (handle_t *,struct inode *,struct inode *);#else /* CONFIG_EXT4_FS_POSIX_ACL */ #include <linux/sched.h> -#define ext4_check_acl NULL +#define ext4_check_acl_rcu NULLstatic inline int ext4_acl_chmod (struct inode *inode)
fs/ext4/file.c +1 -1
@@ -200,7 +200,7 @@ const struct inode_operations ext4_file_inode_operations = { .listxattr = ext4_listxattr, .removexattr = generic_removexattr, #endif - .check_acl = ext4_check_acl, + .check_acl_rcu = ext4_check_acl_rcu, .fallocate = ext4_fallocate, .fiemap = ext4_fiemap, };
fs/ext4/namei.c +2 -2
@@ -2511,7 +2511,7 @@ const struct inode_operations ext4_dir_inode_operations = { .listxattr = ext4_listxattr, .removexattr = generic_removexattr, #endif - .check_acl = ext4_check_acl, + .check_acl_rcu = ext4_check_acl_rcu, .fiemap = ext4_fiemap, };
@@ -2523,5 +2523,5 @@ const struct inode_operations ext4_special_inode_operations = { .listxattr = ext4_listxattr, .removexattr = generic_removexattr, #endif - .check_acl = ext4_check_acl, + .check_acl_rcu = ext4_check_acl_rcu, };
fs/generic_acl.c +20 -0
@@ -190,6 +190,26 @@ generic_acl_chmod(struct inode *inode) }int +generic_check_acl_rcu (struct inode *inode,int mask,unsigned int flags) +{ +if (flags &IPERM_FLAG_RCU ) { +if (!negative_cached_acl (inode,ACL_TYPE_ACCESS )) +return -ECHILD ; + }else { +struct posix_acl *acl; + + acl =get_cached_acl (inode,ACL_TYPE_ACCESS ); + +if (acl) { +int error =posix_acl_permission (inode, acl, mask); +posix_acl_release (acl); +return error; + } + } +return -EAGAIN ; +} + +int generic_check_acl (struct inode *inode,int mask) {struct posix_acl *acl =get_cached_acl (inode,ACL_TYPE_ACCESS );
fs/xfs/linux-2.6/xfs_acl.c +7 -1
@@ -219,7 +219,7 @@ xfs_set_acl(struct inode *inode, int type, struct posix_acl *acl) }int -xfs_check_acl (struct inode *inode,int mask) +xfs_check_acl_rcu (struct inode *inode,int mask,unsigned int flags) {struct xfs_inode *ip =XFS_I (inode);struct posix_acl *acl;
@@ -234,6 +234,12 @@ xfs_check_acl(struct inode *inode, int mask)if (!XFS_IFORK_Q (ip))return -EAGAIN ; +if (flags &IPERM_FLAG_RCU ) { +if (!negative_cached_acl (inode,ACL_TYPE_ACCESS )) +return -ECHILD ; +return -EAGAIN ; + } + acl =xfs_get_acl (inode,ACL_TYPE_ACCESS );if (IS_ERR (acl))return PTR_ERR (acl);
fs/xfs/linux-2.6/xfs_iops.c +4 -4
@@ -643,7 +643,7 @@ xfs_vn_fiemap( }static const struct inode_operations xfs_inode_operations = { - .check_acl = xfs_check_acl, + .check_acl_rcu = xfs_check_acl_rcu, .getattr = xfs_vn_getattr, .setattr = xfs_vn_setattr, .setxattr = generic_setxattr,
@@ -670,7 +670,7 @@ static const struct inode_operations xfs_dir_inode_operations = { .rmdir = xfs_vn_unlink, .mknod = xfs_vn_mknod, .rename = xfs_vn_rename, - .check_acl = xfs_check_acl, + .check_acl_rcu = xfs_check_acl_rcu, .getattr = xfs_vn_getattr, .setattr = xfs_vn_setattr, .setxattr = generic_setxattr,
@@ -695,7 +695,7 @@ static const struct inode_operations xfs_dir_ci_inode_operations = { .rmdir = xfs_vn_unlink, .mknod = xfs_vn_mknod, .rename = xfs_vn_rename, - .check_acl = xfs_check_acl, + .check_acl_rcu = xfs_check_acl_rcu, .getattr = xfs_vn_getattr, .setattr = xfs_vn_setattr, .setxattr = generic_setxattr,
@@ -708,7 +708,7 @@ static const struct inode_operations xfs_symlink_inode_operations = { .readlink = generic_readlink, .follow_link = xfs_vn_follow_link, .put_link = xfs_vn_put_link, - .check_acl = xfs_check_acl, + .check_acl_rcu = xfs_check_acl_rcu, .getattr = xfs_vn_getattr, .setattr = xfs_vn_setattr, .setxattr = generic_setxattr,
fs/xfs/xfs_acl.h +2 -2
@@ -42,7 +42,7 @@ struct xfs_acl {#define SGI_ACL_DEFAULT_SIZE (sizeof(SGI_ACL_DEFAULT)-1)#ifdef CONFIG_XFS_POSIX_ACL -extern int xfs_check_acl (struct inode *inode,int mask); +extern int xfs_check_acl_rcu (struct inode *inode,int mask,unsigned int flags);extern struct posix_acl *xfs_get_acl (struct inode *inode,int type);extern int xfs_inherit_acl (struct inode *inode,struct posix_acl *default_acl);extern int xfs_acl_chmod (struct inode *inode);
@@ -52,7 +52,7 @@ extern int posix_acl_default_exists(struct inode *inode);extern const struct xattr_handler xfs_xattr_acl_access_handler;extern const struct xattr_handler xfs_xattr_acl_default_handler;#else -# define xfs_check_acl NULL +# define xfs_check_acl_rcu NULL# define xfs_get_acl (inode, type) NULL# define xfs_inherit_acl (inode, default_acl) 0# define xfs_acl_chmod (inode) 0
include/linux/generic_acl.h +1 -0
@@ -11,5 +11,6 @@ extern const struct xattr_handler generic_acl_default_handler;int generic_acl_init (struct inode *,struct inode *);int generic_acl_chmod (struct inode *);int generic_check_acl (struct inode *inode,int mask); +int generic_check_acl_rcu (struct inode *inode,int mask,unsigned int flags);#endif /* LINUX_GENERIC_ACL_H */
include/linux/posix_acl.h +19 -0
@@ -108,6 +108,25 @@ static inline struct posix_acl *get_cached_acl(struct inode *inode, int type)return acl; } +static inline int negative_cached_acl (struct inode *inode,int type) +{ +struct posix_acl **p, *acl; +switch (type) { +case ACL_TYPE_ACCESS : + p = &inode->i_acl; +break ; +case ACL_TYPE_DEFAULT : + p = &inode->i_default_acl; +break ; +default : +BUG (); + } + acl =ACCESS_ONCE (*p); +if (acl) +return 0 ; +return 1 ; +} +static inline void set_cached_acl (struct inode *inode,int type,struct posix_acl *acl)
mm/shmem.c +3 -3
@@ -2485,7 +2485,7 @@ static const struct inode_operations shmem_inode_operations = { .getxattr = generic_getxattr, .listxattr = generic_listxattr, .removexattr = generic_removexattr, - .check_acl = generic_check_acl, + .check_acl_rcu = generic_check_acl_rcu,#endif };
@@ -2508,7 +2508,7 @@ static const struct inode_operations shmem_dir_inode_operations = { .getxattr = generic_getxattr, .listxattr = generic_listxattr, .removexattr = generic_removexattr, - .check_acl = generic_check_acl, + .check_acl_rcu = generic_check_acl_rcu,#endif };
@@ -2519,7 +2519,7 @@ static const struct inode_operations shmem_special_inode_operations = { .getxattr = generic_getxattr, .listxattr = generic_listxattr, .removexattr = generic_removexattr, - .check_acl = generic_check_acl, + .check_acl_rcu = generic_check_acl_rcu,#endif };
Signature · 1 line
1.7.1
PATCH 42/46 kernel: add bl_list
In reply to Nick PigginIntroduce a type of hlist that can support the use of the lowest bit in the hlist_head. This will be subsequently used to implement per-bucket bit spinlock for inode and dentry hashes, and may be useful in other cases such as network hashes.
--- include/linux/list_bl.h | 141 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/rculist_bl.h | 128 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 269 insertions(+), 0 deletions(-) create mode 100644 include/linux/list_bl.h create mode 100644 include/linux/rculist_bl.h
include/linux/list_bl.h new file +141 -0
@@ -0,0 +1,141 @@ +#ifndef _LINUX_LIST_BL_H +#define _LINUX_LIST_BL_H + +#include <linux/list.h> +#include <linux/bit_spinlock.h> + +/* +* Special version of lists, where head of the list has a bit spinlock +* in the lowest bit. This is useful for scalable hash tables without +* increasing memory footprint overhead. +* +* For modification operations, the 0 bit of hlist_bl_head->first +* pointer must be set. +*/ + +#if defined(CONFIG_SMP ) || defined(CONFIG_DEBUG_SPINLOCK ) +#define LIST_BL_LOCKMASK 1UL +#else +#define LIST_BL_LOCKMASK 0UL +#endif + +#ifdef CONFIG_DEBUG_LIST +#define LIST_BL_BUG_ON (x) BUG_ON(x) +#else +#define LIST_BL_BUG_ON (x) +#endif + + +struct hlist_bl_head { +struct hlist_bl_node *first; +}; + +struct hlist_bl_node { +struct hlist_bl_node *next, **pprev; +}; +#define INIT_HLIST_BL_HEAD (ptr) \ + ((ptr)->first = NULL) + +static inline void INIT_HLIST_BL_NODE (struct hlist_bl_node *h) +{ + h->next =NULL ; + h->pprev =NULL ; +} + +#define hlist_bl_entry (ptr, type, member) container_of(ptr,type,member) + +static inline int hlist_bl_unhashed (const struct hlist_bl_node *h) +{ +return !h->pprev; +} + +static inline struct hlist_bl_node *hlist_bl_first (struct hlist_bl_head *h) +{ +return (struct hlist_bl_node *) + ((unsigned long )h->first & ~LIST_BL_LOCKMASK ); +} + +static inline void hlist_bl_set_first (struct hlist_bl_head *h, +struct hlist_bl_node *n) +{ +LIST_BL_BUG_ON ((unsigned long )n &LIST_BL_LOCKMASK ); +LIST_BL_BUG_ON (!bit_spin_is_locked (0 , (unsigned long *)&h->first)); + h->first = (struct hlist_bl_node *)((unsigned long )n |LIST_BL_LOCKMASK ); +} + +static inline int hlist_bl_empty (const struct hlist_bl_head *h) +{ +return !((unsigned long )h->first & ~LIST_BL_LOCKMASK ); +} + +static inline void hlist_bl_add_head (struct hlist_bl_node *n, +struct hlist_bl_head *h) +{ +struct hlist_bl_node *first =hlist_bl_first (h); + + n->next = first; +if (first) + first->pprev = &n->next; + n->pprev = &h->first; +hlist_bl_set_first (h, n); +} + +static inline void __hlist_bl_del (struct hlist_bl_node *n) +{ +struct hlist_bl_node *next = n->next; +struct hlist_bl_node **pprev = n->pprev; + +LIST_BL_BUG_ON ((unsigned long )n &LIST_BL_LOCKMASK ); + +/* pprev may be `first`, so be careful not to lose the lock bit */ + *pprev = (struct hlist_bl_node *) + ((unsigned long )next | + ((unsigned long )*pprev &LIST_BL_LOCKMASK )); +if (next) + next->pprev = pprev; +} + +static inline void hlist_bl_del (struct hlist_bl_node *n) +{ +__hlist_bl_del (n); + n->next =LIST_POISON1 ; + n->pprev =LIST_POISON2 ; +} + +static inline void hlist_bl_del_init (struct hlist_bl_node *n) +{ +if (!hlist_bl_unhashed (n)) { +__hlist_bl_del (n); +INIT_HLIST_BL_NODE (n); + } +} + +/** +* hlist_bl_for_each_entry - iterate over list of given type +* @tpos: the type * to use as a loop cursor. +* @pos: the &struct hlist_node to use as a loop cursor. +* @head: the head for your list. +* @member: the name of the hlist_node within the struct. +* +*/ +#define hlist_bl_for_each_entry (tpos, pos, head, member) \ + for (pos = hlist_bl_first(head); \ + pos && \ + ({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1;}); \ + pos = pos->next) + +/** +* hlist_bl_for_each_entry_safe - iterate over list of given type safe against removal of list entry +* @tpos: the type * to use as a loop cursor. +* @pos: the &struct hlist_node to use as a loop cursor. +* @n: another &struct hlist_node to use as temporary storage +* @head: the head for your list. +* @member: the name of the hlist_node within the struct. +*/ +#define hlist_bl_for_each_entry_safe (tpos, pos, n, head, member) \ + for (pos = hlist_bl_first(head); \ + pos && ({ n = pos->next; 1; }) && \ + ({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1;}); \ + pos = n) + +#endif
include/linux/rculist_bl.h new file +128 -0
128 lines not on this page · raw message
Signature · 1 line
1.7.1
PATCH 43/46 bit_spinlock: add required includes
In reply to Nick Piggin--- include/linux/bit_spinlock.h | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-)
include/linux/bit_spinlock.h +4 -0
@@ -1,6 +1,10 @@#ifndef __LINUX_BIT_SPINLOCK_H#define __LINUX_BIT_SPINLOCK_H +#include <linux/kernel.h> +#include <linux/preempt.h> +#include <asm/atomic.h> +/* * bit-based spin_lock() *
Signature · 1 line
1.7.1
PATCH 44/46 fs: dcache per-bucket dcache hash locking
In reply to Nick PigginWe can turn the dcache hash locking from a global dcache_hash_lock into per-bucket locking.
--- fs/dcache.c | 131 ++++++++++++++++++++++++++++++++--------------- fs/super.c | 3 +- include/linux/dcache.h | 23 ++------- include/linux/fs.h | 3 +- 4 files changed, 97 insertions(+), 63 deletions(-)
fs/dcache.c +89 -42
@@ -33,13 +33,15 @@#include <linux/bootmem.h> #include <linux/fs_struct.h> #include <linux/hardirq.h> +#include <linux/bit_spinlock.h> +#include <linux/rculist_bl.h> #include "internal.h" /* * Usage: * dcache_inode_lock protects: * - i_dentry, d_alias, d_inode -* dcache_hash_lock protects: +* dcache_hash_bucket lock protects: * - the dcache hash table * dcache_lru_lock protects: * - the dcache lru lists and counters
@@ -57,7 +59,7 @@* dcache_inode_lock * dentry->d_lock * dcache_lru_lock -* dcache_hash_lock +* dcache_hash_bucket lock * * If there is an ancestor relationship: * dentry->d_parent->...->d_parent->d_lock
@@ -74,13 +76,11 @@ int sysctl_vfs_cache_pressure __read_mostly = 100;EXPORT_SYMBOL_GPL (sysctl_vfs_cache_pressure);__cacheline_aligned_in_smp DEFINE_SPINLOCK (dcache_inode_lock ); -__cacheline_aligned_in_smp DEFINE_SPINLOCK (dcache_hash_lock );static __cacheline_aligned_in_smp DEFINE_SPINLOCK (dcache_lru_lock );__cacheline_aligned_in_smp DEFINE_SEQLOCK (rename_lock );EXPORT_SYMBOL (rename_lock);EXPORT_SYMBOL (dcache_inode_lock); -EXPORT_SYMBOL (dcache_hash_lock);static struct kmem_cache *dentry_cache__read_mostly ;
@@ -97,13 +97,35 @@ static struct kmem_cache *dentry_cache __read_mostly;static unsigned int d_hash_mask__read_mostly ;static unsigned int d_hash_shift__read_mostly ; -static struct hlist_head *dentry_hashtable__read_mostly ; + +struct dcache_hash_bucket { +struct hlist_bl_head head; +}; +static struct dcache_hash_bucket *dentry_hashtable__read_mostly ;/* Statistics gathering. */ struct dentry_stat_t dentry_stat = { .age_limit =45 , }; +static inline struct dcache_hash_bucket *d_hash (struct dentry *parent, +unsigned long hash) +{ + hash += ((unsigned long ) parent ^GOLDEN_RATIO_PRIME ) /L1_CACHE_BYTES ; + hash = hash ^ ((hash ^GOLDEN_RATIO_PRIME ) >>D_HASHBITS ); +return dentry_hashtable + (hash &D_HASHMASK ); +} + +static inline void spin_lock_bucket (struct dcache_hash_bucket *b) +{ +bit_spin_lock (0 , (unsigned long *)b); +} + +static inline void spin_unlock_bucket (struct dcache_hash_bucket *b) +{ +__bit_spin_unlock (0 , (unsigned long *)b); +} + static struct percpu_counter nr_dentry __cacheline_aligned_in_smp; static struct percpu_counter nr_dentry_unused __cacheline_aligned_in_smp;
@@ -138,7 +160,7 @@ static void d_free(struct dentry *dentry) dentry->d_op->d_release (dentry);/* if dentry was never inserted into hash, immediate free is OK */ -if (hlist_unhashed (&dentry->d_hash)) +if (hlist_bl_unhashed (&dentry->d_hash))__d_free (&dentry->d_u.d_rcu);else call_rcu (&dentry->d_u.d_rcu, __d_free);
@@ -278,6 +300,39 @@ relock:return d_kill (dentry, parent); } +void __d_drop (struct dentry *dentry) +{ +if (!(dentry->d_flags &DCACHE_UNHASHED )) { +if (unlikely (dentry->d_flags &DCACHE_DISCONNECTED )) { +bit_spin_lock (0 , (unsigned long *)&dentry->d_sb->s_anon); + dentry->d_flags |=DCACHE_UNHASHED ; +hlist_bl_del_init (&dentry->d_hash); +__bit_spin_unlock (0 , (unsigned long *)&dentry->d_sb->s_anon); + }else { +struct dcache_hash_bucket *b; + b =d_hash (dentry->d_parent, dentry->d_name.hash); +spin_lock_bucket (b); +/* +* We may not actually need to put DCACHE_UNHASHED +* manipulations under the hash lock, but follow +* the principle of least surprise. +*/ + dentry->d_flags |=DCACHE_UNHASHED ; +hlist_bl_del_rcu (&dentry->d_hash); +spin_unlock_bucket (b); + } + } +} +EXPORT_SYMBOL (__d_drop); + +void d_drop (struct dentry *dentry) +{ +spin_lock (&dentry->d_lock); +__d_drop (dentry); +spin_unlock (&dentry->d_lock); +} +EXPORT_SYMBOL (d_drop); +/* * This is dput *
@@ -891,8 +946,8 @@ void shrink_dcache_for_umount(struct super_block *sb)spin_unlock (&dentry->d_lock);shrink_dcache_for_umount_subtree (dentry); -while (!hlist_empty (&sb->s_anon)) { - dentry = hlist_entry(sb->s_anon.first, struct dentry, d_hash); +while (!hlist_bl_empty (&sb->s_anon)) { + dentry = hlist_bl_entry(hlist_bl_first(&sb->s_anon), struct dentry, d_hash);shrink_dcache_for_umount_subtree (dentry); } }
@@ -1196,7 +1251,7 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) dentry->d_sb =NULL ; dentry->d_op =NULL ; dentry->d_fsdata =NULL ; -INIT_HLIST_NODE (&dentry->d_hash); +INIT_HLIST_BL_NODE (&dentry->d_hash);INIT_LIST_HEAD (&dentry->d_lru);INIT_LIST_HEAD (&dentry->d_subdirs);INIT_LIST_HEAD (&dentry->d_alias);
@@ -1387,14 +1442,6 @@ struct dentry * d_alloc_root(struct inode * root_inode) }EXPORT_SYMBOL (d_alloc_root); -static inline struct hlist_head *d_hash (struct dentry *parent, -unsigned long hash) -{ - hash += ((unsigned long ) parent ^GOLDEN_RATIO_PRIME ) /L1_CACHE_BYTES ; - hash = hash ^ ((hash ^GOLDEN_RATIO_PRIME ) >>D_HASHBITS ); -return dentry_hashtable + (hash &D_HASHMASK ); -} -/** * d_obtain_alias - find or allocate a dentry for a given inode * @inode: inode to allocate the dentry for
@@ -1449,11 +1496,11 @@ struct dentry *d_obtain_alias(struct inode *inode) tmp->d_sb = inode->i_sb; tmp->d_inode = inode; tmp->d_flags |=DCACHE_DISCONNECTED ; - tmp->d_flags &= ~DCACHE_UNHASHED ;list_add (&tmp->d_alias, &inode->i_dentry); -spin_lock (&dcache_hash_lock); -hlist_add_head (&tmp->d_hash, &inode->i_sb->s_anon); -spin_unlock (&dcache_hash_lock); +bit_spin_lock (0 , (unsigned long *)&tmp->d_sb->s_anon); + tmp->d_flags &= ~DCACHE_UNHASHED ; +hlist_bl_add_head (&tmp->d_hash, &tmp->d_sb->s_anon); +__bit_spin_unlock (0 , (unsigned long *)&tmp->d_sb->s_anon);spin_unlock (&tmp->d_lock);spin_unlock (&dcache_inode_lock);
@@ -1617,8 +1664,8 @@ struct dentry *__d_lookup_rcu(struct dentry *parent, struct qstr *name,unsigned int len = name->len;unsigned int hash = name->hash;const unsigned char *str = name->name; -struct hlist_head *head =d_hash (parent,hash); -struct hlist_node *node; +struct dcache_hash_bucket *b =d_hash (parent, hash); +struct hlist_bl_node *node;struct dentry *dentry;/*
@@ -1641,7 +1688,7 @@ struct dentry *__d_lookup_rcu(struct dentry *parent, struct qstr *name,* * See Documentation/vfs/dcache-locking.txt for more details. */ -hlist_for_each_entry_rcu (dentry, node, head, d_hash) { +hlist_bl_for_each_entry_rcu (dentry, node, &b->head, d_hash) {struct inode *i;const char *tname;int tlen;
@@ -1754,8 +1801,8 @@ struct dentry *__d_lookup(struct dentry *parent, struct qstr *name)unsigned int len = name->len;unsigned int hash = name->hash;const unsigned char *str = name->name; -struct hlist_head *head =d_hash (parent,hash); -struct hlist_node *node; +struct dcache_hash_bucket *b =d_hash (parent, hash); +struct hlist_bl_node *node;struct dentry *found =NULL ;struct dentry *dentry;
@@ -1781,7 +1828,7 @@ struct dentry *__d_lookup(struct dentry *parent, struct qstr *name)*/ rcu_read_lock (); -hlist_for_each_entry_rcu (dentry, node, head, d_hash) { +hlist_bl_for_each_entry_rcu (dentry, node, &b->head, d_hash) {const char *tname;int tlen;
@@ -1932,11 +1979,12 @@ again: }EXPORT_SYMBOL (d_delete); -static void __d_rehash (struct dentry * entry,struct hlist_head *list) +static void __d_rehash (struct dentry * entry,struct dcache_hash_bucket *b) { - +spin_lock_bucket (b); entry->d_flags &= ~DCACHE_UNHASHED ; -hlist_add_head_rcu (&entry->d_hash, list); +hlist_bl_add_head_rcu (&entry->d_hash, &b->head); +spin_unlock_bucket (b); }static void _d_rehash (struct dentry * entry)
@@ -1954,9 +2002,7 @@ static void _d_rehash(struct dentry * entry)void d_rehash (struct dentry * entry) {spin_lock (&entry->d_lock); -spin_lock (&dcache_hash_lock);_d_rehash (entry); -spin_unlock (&dcache_hash_lock);spin_unlock (&entry->d_lock); }EXPORT_SYMBOL (d_rehash);
@@ -2035,6 +2081,7 @@ static void switch_names(struct dentry *dentry, struct dentry *target)*/ void d_move (struct dentry * dentry,struct dentry * target) { +struct dcache_hash_bucket *b;if (!dentry->d_inode)printk (KERN_WARNING "VFS: moving negative dcache entry\n" );
@@ -2065,11 +2112,13 @@ void d_move(struct dentry * dentry, struct dentry * target) }/* Move the dentry to the target hash queue, if on different bucket */ -spin_lock (&dcache_hash_lock); -if (!d_unhashed (dentry)) -hlist_del_rcu (&dentry->d_hash); +if (!d_unhashed (dentry)) { + b =d_hash (dentry->d_parent, dentry->d_name.hash); +spin_lock_bucket (b); +hlist_bl_del_rcu (&dentry->d_hash); +spin_unlock_bucket (b); + }__d_rehash (dentry,d_hash (target->d_parent, target->d_name.hash)); -spin_unlock (&dcache_hash_lock);/* Unhash the target: dput() will then get rid of it */ __d_drop (target);
@@ -2280,9 +2329,7 @@ struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode)spin_lock (&actual->d_lock); found: -spin_lock (&dcache_hash_lock);_d_rehash (actual); -spin_unlock (&dcache_hash_lock);spin_unlock (&actual->d_lock);spin_unlock (&dcache_inode_lock); out_nolock:
@@ -2864,7 +2911,7 @@ static void __init dcache_init_early(void) dentry_hashtable = alloc_large_system_hash("Dentry cache", - sizeof(struct hlist_head), + sizeof(struct dcache_hash_bucket), dhash_entries, 13, HASH_EARLY,
@@ -2873,7 +2920,7 @@ static void __init dcache_init_early(void) 0);for (loop =0 ; loop < (1 << d_hash_shift); loop++) -INIT_HLIST_HEAD (&dentry_hashtable[loop]); +INIT_HLIST_BL_HEAD (&dentry_hashtable[loop].head); }static void __init dcache_init (void )
@@ -2899,7 +2946,7 @@ static void __init dcache_init(void) dentry_hashtable = alloc_large_system_hash("Dentry cache", - sizeof(struct hlist_head), + sizeof(struct dcache_hash_bucket), dhash_entries, 13, 0,
@@ -2908,7 +2955,7 @@ static void __init dcache_init(void) 0);for (loop =0 ; loop < (1 << d_hash_shift); loop++) -INIT_HLIST_HEAD (&dentry_hashtable[loop]); +INIT_HLIST_BL_HEAD (&dentry_hashtable[loop].head); }/* SLAB cache for __getname() consumers */
fs/super.c +2 -1
@@ -30,6 +30,7 @@#include <linux/idr.h> #include <linux/mutex.h> #include <linux/backing-dev.h> +#include <linux/rculist_bl.h> #include "internal.h"
@@ -71,7 +72,7 @@ static struct super_block *alloc_super(struct file_system_type *type)INIT_LIST_HEAD (&s->s_files);#endif INIT_LIST_HEAD (&s->s_instances); -INIT_HLIST_HEAD (&s->s_anon); +INIT_HLIST_BL_HEAD (&s->s_anon);INIT_LIST_HEAD (&s->s_inodes);INIT_LIST_HEAD (&s->s_dentry_lru);init_rwsem (&s->s_umount);
include/linux/dcache.h +4 -19
@@ -4,6 +4,7 @@#include <asm/atomic.h> #include <linux/list.h> #include <linux/rculist.h> +#include <linux/rculist_bl.h> #include <linux/spinlock.h> #include <linux/seqlock.h> #include <linux/cache.h>
@@ -91,7 +92,7 @@ struct dentry {/* RCU lookup touched fields */ unsigned int d_flags;/* protected by d_lock */ seqcount_t d_seq;/* per dentry seqlock */ -struct hlist_node d_hash;/* lookup hash list */ +struct hlist_bl_node d_hash;/* lookup hash list */ struct dentry *d_parent;/* parent directory */ struct qstr d_name;struct inode *d_inode;/* Where the name belongs to - NULL is
@@ -193,7 +194,6 @@ struct dentry_operations { (DCACHE_OP_REVALIDATE|DCACHE_OP_REVALIDATE_RCU)extern spinlock_t dcache_inode_lock; -extern spinlock_t dcache_hash_lock;extern seqlock_t rename_lock;/**
@@ -211,23 +211,8 @@ extern seqlock_t rename_lock;* * __d_drop requires dentry->d_lock. */ - -static inline void __d_drop (struct dentry *dentry) -{ -if (!(dentry->d_flags &DCACHE_UNHASHED )) { - dentry->d_flags |=DCACHE_UNHASHED ; -spin_lock (&dcache_hash_lock); -hlist_del_rcu (&dentry->d_hash); -spin_unlock (&dcache_hash_lock); - } -} - -static inline void d_drop (struct dentry *dentry) -{ -spin_lock (&dentry->d_lock); -__d_drop (dentry); -spin_unlock (&dentry->d_lock); -} +void d_drop (struct dentry *dentry); +void __d_drop (struct dentry *dentry);static inline int dname_external (struct dentry *dentry) {
include/linux/fs.h +2 -1
@@ -392,6 +392,7 @@ struct inodes_stat_t {#include <linux/capability.h> #include <linux/semaphore.h> #include <linux/fiemap.h> +#include <linux/rculist_bl.h> #include <asm/atomic.h> #include <asm/byteorder.h>
@@ -1376,7 +1377,7 @@ struct super_block {const struct xattr_handler **s_xattr;struct list_head s_inodes;/* all inodes */ -struct hlist_head s_anon;/* anonymous dentries for (nfs) exporting */ +struct hlist_bl_head s_anon;/* anonymous dentries for (nfs) exporting */ #ifdef CONFIG_SMP struct list_head __percpu *s_files;#else
Signature · 1 line
1.7.1
PATCH 45/46 fs: dcache per-inode inode alias locking
In reply to Nick Piggindcache_inode_lock can be replaced with per-inode locking. Use existing inode->i_lock for this. This is slightly non-trivial because we sometimes need to find the inode from the dentry, which requires d_inode to be stabilised (either with refcount or d_lock).
--- fs/9p/vfs_inode.c | 4 +- fs/affs/amigaffs.c | 4 +- fs/dcache.c | 84 +++++++++++++++++++++++++---------------------- fs/exportfs/expfs.c | 12 ++++--- fs/nfs/getroot.c | 4 +- fs/notify/fsnotify.c | 4 +- fs/ocfs2/dcache.c | 4 +- include/linux/dcache.h | 1 - 8 files changed, 62 insertions(+), 55 deletions(-)
fs/9p/vfs_inode.c +2 -2
@@ -277,11 +277,11 @@ static struct dentry *v9fs_dentry_from_dir_inode(struct inode *inode) {struct dentry *dentry; -spin_lock (&dcache_inode_lock); +spin_lock (&inode->i_lock);/* Directory should have only one entry. */ BUG_ON (S_ISDIR (inode->i_mode) && !list_is_singular (&inode->i_dentry)); dentry = list_entry(inode->i_dentry.next, struct dentry, d_alias); -spin_unlock (&dcache_inode_lock); +spin_unlock (&inode->i_lock);return dentry; }
fs/affs/amigaffs.c +2 -2
@@ -128,7 +128,7 @@ affs_fix_dcache(struct dentry *dentry, u32 entry_ino)void *data = dentry->d_fsdata;struct list_head *head, *next; -spin_lock (&dcache_inode_lock); +spin_lock (&inode->i_lock); head = &inode->i_dentry; next = head->next;while (next != head) {
@@ -139,7 +139,7 @@ affs_fix_dcache(struct dentry *dentry, u32 entry_ino) } next = next->next; } -spin_unlock (&dcache_inode_lock); +spin_unlock (&inode->i_lock); }
fs/dcache.c +45 -39
271 lines not on this page · raw message
fs/exportfs/expfs.c +7 -5
@@ -43,24 +43,26 @@ find_acceptable_alias(struct dentry *result, void *context) {struct dentry *dentry, *toput =NULL ; +struct inode *inode;if (acceptable (context, result))return result; -spin_lock (&dcache_inode_lock); -list_for_each_entry (dentry, &result->d_inode->i_dentry, d_alias) { + inode = result->d_inode; +spin_lock (&inode->i_lock); +list_for_each_entry (dentry, &inode->i_dentry, d_alias) {dget (dentry); -spin_unlock (&dcache_inode_lock); +spin_unlock (&inode->i_lock);if (toput)dput (toput);if (dentry != result &&acceptable (context, dentry)) {dput (result);return dentry; } -spin_lock (&dcache_inode_lock); +spin_lock (&inode->i_lock); toput = dentry; } -spin_unlock (&dcache_inode_lock); +spin_unlock (&inode->i_lock);if (toput)dput (toput);
fs/nfs/getroot.c +2 -2
@@ -63,11 +63,11 @@ static int nfs_superblock_set_dummy_root(struct super_block *sb, struct inode *i* This again causes shrink_dcache_for_umount_subtree() to * Oops, since the test for IS_ROOT() will fail. */ -spin_lock (&dcache_inode_lock); +spin_lock (&sb->s_root->d_inode->i_lock);spin_lock (&sb->s_root->d_lock);list_del_init (&sb->s_root->d_alias);spin_unlock (&sb->s_root->d_lock); -spin_unlock (&dcache_inode_lock); +spin_unlock (&sb->s_root->d_inode->i_lock); }return 0 ; }
fs/notify/fsnotify.c +2 -2
@@ -59,7 +59,7 @@ void __fsnotify_update_child_dentry_flags(struct inode *inode)/* determine if the children should tell inode about their events */ watched =fsnotify_inode_watches_children (inode); -spin_lock (&dcache_inode_lock); +spin_lock (&inode->i_lock);/* run all of the dentries associated with this inode. Since this is a * directory, there damn well better only be one item on this list */ list_for_each_entry (alias, &inode->i_dentry, d_alias) {
@@ -82,7 +82,7 @@ void __fsnotify_update_child_dentry_flags(struct inode *inode) }spin_unlock (&alias->d_lock); } -spin_unlock (&dcache_inode_lock); +spin_unlock (&inode->i_lock); }/* Notify this dentry's parent about a child's events. */
fs/ocfs2/dcache.c +2 -2
@@ -169,7 +169,7 @@ struct dentry *ocfs2_find_local_alias(struct inode *inode,struct list_head *p;struct dentry *dentry =NULL ; -spin_lock (&dcache_inode_lock); +spin_lock (&inode->i_lock);list_for_each (p, &inode->i_dentry) { dentry = list_entry(p, struct dentry, d_alias);
@@ -187,7 +187,7 @@ struct dentry *ocfs2_find_local_alias(struct inode *inode, dentry =NULL ; } -spin_unlock (&dcache_inode_lock); +spin_unlock (&inode->i_lock);return dentry; }
include/linux/dcache.h +0 -1
@@ -193,7 +193,6 @@ struct dentry_operations {#define DCACHE_OP_REVALIDATE_EITHER \ (DCACHE_OP_REVALIDATE|DCACHE_OP_REVALIDATE_RCU) -extern spinlock_t dcache_inode_lock;extern seqlock_t rename_lock;/**
Signature · 1 line
1.7.1
PATCH 46/46 fs: improve scalability of pseudo filesystems
In reply to Nick PigginRegardless of how much we possibly try to scale dcache, there is likely always going to be some fundamental contention when adding or removing children under the same parent. Pseudo filesystems do not seem need to have connected dentries because by definition they are disconnected.
--- fs/anon_inodes.c | 2 +- fs/dcache.c | 12 ++++++++++++ fs/pipe.c | 2 +- include/linux/dcache.h | 1 + net/socket.c | 2 +- 5 files changed, 16 insertions(+), 3 deletions(-)
fs/anon_inodes.c +1 -1
@@ -102,7 +102,7 @@ struct file *anon_inode_getfile(const char *name, this.name = name; this.len =strlen (name); this.hash =0 ; - path.dentry =d_alloc (anon_inode_mnt->mnt_sb->s_root, &this); + path.dentry =d_alloc_pseudo (anon_inode_mnt->mnt_sb, &this);if (!path.dentry) goto err_module;
fs/dcache.c +12 -0
@@ -1276,6 +1276,18 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) }EXPORT_SYMBOL (d_alloc); +struct dentry *d_alloc_pseudo (struct super_block *sb,const struct qstr *name) +{ +struct dentry *dentry =d_alloc (NULL , name); +if (dentry) { + dentry->d_sb = sb; + dentry->d_parent = dentry; + dentry->d_flags |=DCACHE_DISCONNECTED ; + } +return dentry; +} +EXPORT_SYMBOL (d_alloc_pseudo); +struct dentry *d_alloc_name (struct dentry *parent,const char *name) {struct qstr q;
fs/pipe.c +1 -1
@@ -999,7 +999,7 @@ struct file *create_write_pipe(int flags) goto err; err = -ENOMEM ; - path.dentry =d_alloc (pipe_mnt->mnt_sb->s_root, &name); + path.dentry =d_alloc_pseudo (pipe_mnt->mnt_sb, &name);if (!path.dentry) goto err_inode; path.mnt =mntget (pipe_mnt);
include/linux/dcache.h +1 -0
@@ -229,6 +229,7 @@ extern void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op/* allocate/de-allocate */ extern struct dentry *d_alloc (struct dentry *,const struct qstr *); +extern struct dentry *d_alloc_pseudo (struct super_block *,const struct qstr *);extern struct dentry *d_splice_alias (struct inode *,struct dentry *);extern struct dentry *d_add_ci (struct dentry *,struct inode *,struct qstr *);extern struct dentry *d_obtain_alias (struct inode *);
net/socket.c +1 -1
@@ -361,7 +361,7 @@ static int sock_alloc_file(struct socket *sock, struct file **f, int flags)if (unlikely (fd <0 ))return fd; - path.dentry =d_alloc (sock_mnt->mnt_sb->s_root, &name); + path.dentry =d_alloc_pseudo (sock_mnt->mnt_sb, &name);if (unlikely (!path.dentry)) {put_unused_fd (fd);return -ENOMEM ;
Signature · 1 line
1.7.1
Quoted 2 lines · Nick Piggin
git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.gitvfs-scale-working
I get:
CC [M] fs/cifs/inode.ofs/cifs/inode.c: In function ‘inode_has_hashed_dentries’:fs/cifs/inode.c:807: error: ‘dcache_inode_lock’ undeclared (first use in this function)fs/cifs/inode.c:807: error: (Each undeclared identifier is reported only once fs/cifs/inode.c:807: error: for each function it appears in.) make[3]: *** [fs/cifs/inode.o] Error 1
make[2]: *** [fs/cifs] Error 2
make[1]: *** [fs] Error 2
I used the latest mainline.
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 15 lines · Anca Emanuel
Quoted 2 lines · Nick Piggin
git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.gitvfs-scale-workingI get:
CC [M]fs/cifs/inode.ofs/cifs/inode.c: In function ‘inode_has_hashed_dentries’:fs/cifs/inode.c:807: error: ‘dcache_inode_lock’ undeclared (first use in this function)fs/cifs/inode.c:807: error: (Each undeclared identifier is reported only oncefs/cifs/inode.c:807: error: for each function it appears in.) make[3]: *** [fs/cifs/inode.o] Error 1
make[2]: *** [fs/cifs] Error 2
make[1]: *** [fs] Error 2
Quoted 1 line
I used the latest mainline.
Sorry, missed a conversion, it just needs to be changed to inode->i_lock. Pushed the fix to git.
Thanks,
Nick
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 16 lines · Nick Piggin
Quoted 15 lines · Anca Emanuel
Quoted 2 lines · Nick Piggin
git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.gitvfs-scale-workingI get:
CC [M] fs/cifs/inode.ofs/cifs/inode.c: In function ‘inode_has_hashed_dentries’:fs/cifs/inode.c:807: error: ‘dcache_inode_lock’ undeclared (first use in this function)fs/cifs/inode.c:807: error: (Each undeclared identifier is reported only oncefs/cifs/inode.c:807: error: for each function it appears in.) make[3]: *** [fs/cifs/inode.o] Error 1
make[2]: *** [fs/cifs] Error 2
make[1]: *** [fs] Error 2
Quoted 5 lines
Quoted 1 line
I used the latest mainline.
Sorry, missed a conversion, it just needs to be changed to
inode->i_lock. Pushed the fix to git.
I attached a patch to my posting in [1] but it was somehow "eaten" (here in my mbox the patch is definitely attached). Patchwork is also not listing my patch.
Anyway, it's fixed - that's good.
I have to check why I get a Call trace with systemd-v15 but not with sysvinit package here on Debian.
- Sedat -
[1] http://lkml.org/lkml/2010/11/27/145
fs/cifs/inode.c +3 -3
@@ -804,14 +804,14 @@ inode_has_hashed_dentries(struct inode * {struct dentry *dentry; -spin_lock (&dcache_inode_lock); +spin_lock (&inode->i_lock);list_for_each_entry (dentry, &inode->i_dentry, d_alias) {if (!d_unhashed (dentry) ||IS_ROOT (dentry)) { -spin_unlock (&dcache_inode_lock); +spin_unlock (&inode->i_lock);return true; } } -spin_unlock (&dcache_inode_lock); +spin_unlock (&inode->i_lock);return false; }
David Miller
maintainerFrom: Nick Piggin <[email protected]>
Date: Sat, 27 Nov 2010 21:15:58 +1100
Quoted 1 line
git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.gitvfs-scale-working
Just want to say that I've been running this code for the past few days on my 128 cpu box and it seems quite sturdy.
If there are any kinds of vfs benchmarks you want me to run on this machine both with and without the scaling changes, just let me know.
Quoted 10 lines · David Miller
From: Nick Piggin <[email protected]>
Date: Sat, 27 Nov 2010 21:15:58 +1100Quoted 1 line
git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.gitvfs-scale-workingJust want to say that I've been running this code for the past few days on my 128 cpu box and it seems quite sturdy.
If there are any kinds of vfs benchmarks you want me to run on this machine both with and without the scaling changes, just let me know.
Hi Dave, great, thanks!
I am mostly interested in single-thread performance on different ISAs and different microarchitectures.
Microbenchmarks are easy, I attached a couple of quick ones I use in an offline mail.
Single-threaded (preloadindex=false), cached and uncached, git diff on an unmodified tree is slightly more real world.
And then anything else you can think of.
Thanks,
Nick
Quoted 11 lines · Nick Piggin
git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.gitvfs-scale-workingHere is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:
* preparation patches
* introduce new locks to take overdcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking steps
Just got this set of traces doing an 8-way parallel remove of 50 million inodes at about 40M inodes unlinked:
[ 5954.061633] BUG: sleeping function called from invalid context at arch/x86/mm/fault.c:1081
[ 5954.062466] in_atomic(): 0, irqs_disabled(): 1, pid: 2927, name: rm
[ 5954.063122] 3 locks held by rm/2927:
[ 5954.063476] #0: (&sb->s_type->i_mutex_key#12/1){+.+.+.}, at: [<ffffffff8116f5e1>] do_rmdir+0x81/0x130
[ 5954.064014] #1: (&sb->s_type->i_mutex_key#12){+.+.+.}, at: [<ffffffff8116d3a8>] vfs_rmdir+0x58/0xe0
[ 5954.064014] #2: (rcu_read_lock){.+.+..}, at: [<ffffffff811779c0>] shrink_dentry_list+0x0/0x430
[ 5954.064014] irq event stamp: 1484376719
[ 5954.064014] hardirqs last enabled at (1484376719): [<ffffffff810ebf07>] __call_rcu+0xd7/0x1a0
[ 5954.064014] hardirqs last disabled at (1484376718): [<ffffffff810ebe7a>] __call_rcu+0x4a/0x1a0
[ 5954.064014] softirqs last enabled at (1484376586): [<ffffffff8108b911>] __do_softirq+0x161/0x270
[ 5954.064014] softirqs last disabled at (1484376581): [<ffffffff8103af1c>] call_softirq+0x1c/0x50
[ 5954.064014] Pid: 2927, comm: rm Not tainted 2.6.37-rc4-dgc+ #794
[ 5954.064014] Call Trace:
[ 5954.064014] [<ffffffff810b95b0>] ? print_irqtrace_events+0xd0/0xe0
[ 5954.064014] [<ffffffff81076455>] __might_sleep+0xf5/0x130
[ 5954.064014] [<ffffffff81b1e603>] do_page_fault+0x103/0x4f0
[ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014] [<ffffffff81b19b28>] ? trace_hardirqs_off_thunk+0x3a/0x3c
[ 5954.064014] [<ffffffff810b8e20>] ? usage_match+0x0/0x20
[ 5954.064014] [<ffffffff81b1af25>] page_fault+0x25/0x30
[ 5954.064014] [<ffffffff810b8e20>] ? usage_match+0x0/0x20
[ 5954.064014] [<ffffffff810ba1c8>] ? __bfs+0xc8/0x260
[ 5954.064014] [<ffffffff810ba123>] ? __bfs+0x23/0x260
[ 5954.064014] [<ffffffff810ba4d2>] find_usage_backwards+0x42/0x80
[ 5954.064014] [<ffffffff810bcec4>] check_usage_backwards+0x64/0xf0
[ 5954.064014] [<ffffffff8104796f>] ? save_stack_trace+0x2f/0x50
[ 5954.064014] [<ffffffff810bce60>] ? check_usage_backwards+0x0/0xf0
[ 5954.064014] [<ffffffff810bd9a9>] mark_lock+0x1a9/0x440
[ 5954.064014] [<ffffffff810be989>] __lock_acquire+0x5a9/0x14b0
[ 5954.064014] [<ffffffff810be716>] ? __lock_acquire+0x336/0x14b0
[ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014] [<ffffffff810bf944>] lock_acquire+0xb4/0x140
[ 5954.064014] [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.064014] [<ffffffff81b19d86>] _raw_spin_lock+0x36/0x70
[ 5954.064014] [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.064014] [<ffffffff81177a1c>] shrink_dentry_list+0x5c/0x430
[ 5954.064014] [<ffffffff811779c0>] ? shrink_dentry_list+0x0/0x430
[ 5954.064014] [<ffffffff816b9c7e>] ? do_raw_spin_unlock+0x5e/0xb0
[ 5954.064014] [<ffffffff81177f2d>] __shrink_dcache_sb+0x13d/0x1c0
[ 5954.064014] [<ffffffff811784bf>] shrink_dcache_parent+0x32f/0x390
[ 5954.064014] [<ffffffff8116d31d>] dentry_unhash+0x3d/0x70
[ 5954.064014] [<ffffffff8116d3b0>] vfs_rmdir+0x60/0xe0
[ 5954.064014] [<ffffffff8116f673>] do_rmdir+0x113/0x130
[ 5954.064014] [<ffffffff8103a03a>] ? sysret_check+0x2e/0x69
[ 5954.064014] [<ffffffff81b19ae9>] ? trace_hardirqs_on_thunk+0x3a/0x3f
[ 5954.064014] [<ffffffff8116f6c5>] sys_unlinkat+0x35/0x40
[ 5954.064014] [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
[ 5954.092916] BUG: unable to handle kernel NULL pointer dereference at (null)
[ 5954.093806] IP: [<ffffffff810ba1c8>] __bfs+0xc8/0x260
[ 5954.094331] PGD 1084e5067 PUD 102368067 PMD 0
[ 5954.094830] Oops: 0000 [#1] SMP
[ 5954.095194] last sysfs file: /sys/devices/system/cpu/online
[ 5954.095760] CPU 6
[ 5954.095954] Modules linked in:
[ 5954.096319]
[ 5954.096483] Pid: 2927, comm: rm Not tainted 2.6.37-rc4-dgc+ #794 /Bochs
[ 5954.096665] RIP: 0010:[<ffffffff810ba1c8>] [<ffffffff810ba1c8>] __bfs+0xc8/0x260
[ 5954.096665] RSP: 0018:ffff8801175539a8 EFLAGS: 00010046
[ 5954.096665] RAX: ffffffff8267d980 RBX: ffffffff8267d980 RCX: ffff880117553a48
[ 5954.096665] RDX: ffff8801175539d0 RSI: 0000000000000000 RDI: ffff880117553a48
[ 5954.096665] RBP: ffff880117553a08 R08: 0000000000000000 R09: 0000000000000000
[ 5954.096665] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000000
[ 5954.096665] R13: ffffffff810b8e20 R14: ffff880117553a90 R15: 0000000000000000
[ 5954.096665] FS: 00007f4594cf3700(0000) GS:ffff8800dfa00000(0000) knlGS:0000000000000000
[ 5954.096665] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[ 5954.096665] CR2: 00007f2f21e89c60 CR3: 0000000110b0f000 CR4: 00000000000006e0
[ 5954.096665] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 5954.096665] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[ 5954.096665] Process rm (pid: 2927, threadinfo ffff880117552000, task ffff88010ff10b00)
[ 5954.096665] Stack:
[ 5954.096665] ffffffff8267d868 00007fffa02d1428 ffff8800ffffffff ffff880100000000
[ 5954.096665] 000000000000b720 ffff880117553a48 ffffffff8267d868 ffff880117553a48
[ 5954.096665] 0000000000000000 ffff88010ff10b00 0000000000000000 ffffffff81dacba0
[ 5954.096665] Call Trace:
[ 5954.096665] [<ffffffff810ba4d2>] find_usage_backwards+0x42/0x80
[ 5954.096665] [<ffffffff810bcec4>] check_usage_backwards+0x64/0xf0
[ 5954.096665] [<ffffffff8104796f>] ? save_stack_trace+0x2f/0x50
[ 5954.096665] [<ffffffff810bce60>] ? check_usage_backwards+0x0/0xf0
[ 5954.096665] [<ffffffff810bd9a9>] mark_lock+0x1a9/0x440
[ 5954.096665] [<ffffffff810be989>] __lock_acquire+0x5a9/0x14b0
[ 5954.096665] [<ffffffff810be716>] ? __lock_acquire+0x336/0x14b0
[ 5954.096665] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.096665] [<ffffffff810bf944>] lock_acquire+0xb4/0x140
[ 5954.096665] [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.096665] [<ffffffff81b19d86>] _raw_spin_lock+0x36/0x70
[ 5954.096665] [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.096665] [<ffffffff81177a1c>] shrink_dentry_list+0x5c/0x430
[ 5954.096665] [<ffffffff811779c0>] ? shrink_dentry_list+0x0/0x430
[ 5954.096665] [<ffffffff816b9c7e>] ? do_raw_spin_unlock+0x5e/0xb0
[ 5954.096665] [<ffffffff81177f2d>] __shrink_dcache_sb+0x13d/0x1c0
[ 5954.096665] [<ffffffff811784bf>] shrink_dcache_parent+0x32f/0x390
[ 5954.096665] [<ffffffff8116d31d>] dentry_unhash+0x3d/0x70
[ 5954.096665] [<ffffffff8116d3b0>] vfs_rmdir+0x60/0xe0
[ 5954.096665] [<ffffffff8116f673>] do_rmdir+0x113/0x130
[ 5954.096665] [<ffffffff8103a03a>] ? sysret_check+0x2e/0x69
[ 5954.096665] [<ffffffff81b19ae9>] ? trace_hardirqs_on_thunk+0x3a/0x3f
[ 5954.096665] [<ffffffff8116f6c5>] sys_unlinkat+0x35/0x40
[ 5954.096665] [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
[ 5954.096665] Code: 0a 89 05 dc 0f a8 01 48 8b 41 10 48 85 c0 0f 84 1f 01 00 00 48 8d 98 70 01 00 00 48 05 80 01 00 00 45 85 c0 48 0f 44 d8 4c 8b 3b <49> 8b 07 49 39
[ 5954.096665] RIP [<ffffffff810ba1c8>] __bfs+0xc8/0x260
[ 5954.096665] RSP <ffff8801175539a8>
[ 5954.096665] CR2: 0000000000000000
[ 5954.127991] ---[ end trace 85a6727c2d4e3d90 ]---Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
Nick Piggin
reviewerQuoted 7 lines · Dave Chinner
Quoted 6 lines · Nick Piggin
git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.gitvfs-scale-workingHere is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:
Quoted 8 lines
Quoted 5 lines
* preparation patches
* introduce new locks to take overdcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking stepsJust got this set of traces doing an 8-way parallel remove of 50 million inodes at about 40M inodes unlinked:
Thanks for testing...
Quoted 41 lines
[ 5954.061633] BUG: sleeping function called from invalid context at arch/x86/mm/fault.c:1081
[ 5954.062466] in_atomic(): 0, irqs_disabled(): 1, pid: 2927, name: rm
[ 5954.063122] 3 locks held by rm/2927:
[ 5954.063476] #0: (&sb->s_type->i_mutex_key#12/1){+.+.+.}, at: [<ffffffff8116f5e1>] do_rmdir+0x81/0x130
[ 5954.064014] #1: (&sb->s_type->i_mutex_key#12){+.+.+.}, at: [<ffffffff8116d3a8>] vfs_rmdir+0x58/0xe0
[ 5954.064014] #2: (rcu_read_lock){.+.+..}, at: [<ffffffff811779c0>] shrink_dentry_list+0x0/0x430
[ 5954.064014] irq event stamp: 1484376719
[ 5954.064014] hardirqs last enabled at (1484376719): [<ffffffff810ebf07>] __call_rcu+0xd7/0x1a0
[ 5954.064014] hardirqs last disabled at (1484376718): [<ffffffff810ebe7a>] __call_rcu+0x4a/0x1a0
[ 5954.064014] softirqs last enabled at (1484376586): [<ffffffff8108b911>] __do_softirq+0x161/0x270
[ 5954.064014] softirqs last disabled at (1484376581): [<ffffffff8103af1c>] call_softirq+0x1c/0x50
[ 5954.064014] Pid: 2927, comm: rm Not tainted 2.6.37-rc4-dgc+ #794
[ 5954.064014] Call Trace:
[ 5954.064014] [<ffffffff810b95b0>] ? print_irqtrace_events+0xd0/0xe0
[ 5954.064014] [<ffffffff81076455>] __might_sleep+0xf5/0x130
[ 5954.064014] [<ffffffff81b1e603>] do_page_fault+0x103/0x4f0
[ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014] [<ffffffff81b19b28>] ? trace_hardirqs_off_thunk+0x3a/0x3c
[ 5954.064014] [<ffffffff810b8e20>] ? usage_match+0x0/0x20
[ 5954.064014] [<ffffffff81b1af25>] page_fault+0x25/0x30
[ 5954.064014] [<ffffffff810b8e20>] ? usage_match+0x0/0x20
[ 5954.064014] [<ffffffff810ba1c8>] ? __bfs+0xc8/0x260
[ 5954.064014] [<ffffffff810ba123>] ? __bfs+0x23/0x260
[ 5954.064014] [<ffffffff810ba4d2>] find_usage_backwards+0x42/0x80
[ 5954.064014] [<ffffffff810bcec4>] check_usage_backwards+0x64/0xf0
[ 5954.064014] [<ffffffff8104796f>] ? save_stack_trace+0x2f/0x50
[ 5954.064014] [<ffffffff810bce60>] ? check_usage_backwards+0x0/0xf0
[ 5954.064014] [<ffffffff810bd9a9>] mark_lock+0x1a9/0x440
[ 5954.064014] [<ffffffff810be989>] __lock_acquire+0x5a9/0x14b0
[ 5954.064014] [<ffffffff810be716>] ? __lock_acquire+0x336/0x14b0
[ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014] [<ffffffff810bf944>] lock_acquire+0xb4/0x140
[ 5954.064014] [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.064014] [<ffffffff81b19d86>] _raw_spin_lock+0x36/0x70
[ 5954.064014] [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.064014] [<ffffffff81177a1c>] shrink_dentry_list+0x5c/0x430
[ 5954.064014] [<ffffffff811779c0>] ? shrink_dentry_list+0x0/0x430
[ 5954.064014] [<ffffffff816b9c7e>] ? do_raw_spin_unlock+0x5e/0xb0
[ 5954.064014] [<ffffffff81177f2d>] __shrink_dcache_sb+0x13d/0x1c0Quoted 8 lines
[ 5954.064014] [<ffffffff811784bf>] shrink_dcache_parent+0x32f/0x390 [ 5954.064014] [<ffffffff8116d31d>] dentry_unhash+0x3d/0x70 [ 5954.064014] [<ffffffff8116d3b0>] vfs_rmdir+0x60/0xe0 [ 5954.064014] [<ffffffff8116f673>] do_rmdir+0x113/0x130 [ 5954.064014] [<ffffffff8103a03a>] ? sysret_check+0x2e/0x69 [ 5954.064014] [<ffffffff81b19ae9>] ? trace_hardirqs_on_thunk+0x3a/0x3f [ 5954.064014] [<ffffffff8116f6c5>] sys_unlinkat+0x35/0x40 [ 5954.064014] [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
Seems that lockdep exploded.
Quoted 48 lines
[ 5954.092916] BUG: unable to handle kernel NULL pointer dereference at (null) [ 5954.093806] IP: [<ffffffff810ba1c8>] __bfs+0xc8/0x260 [ 5954.094331] PGD 1084e5067 PUD 102368067 PMD 0 [ 5954.094830] Oops: 0000 [#1] SMP [ 5954.095194] last sysfs file: /sys/devices/system/cpu/online [ 5954.095760] CPU 6 [ 5954.095954] Modules linked in: [ 5954.096319] [ 5954.096483] Pid: 2927, comm: rm Not tainted 2.6.37-rc4-dgc+ #794 /Bochs [ 5954.096665] RIP: 0010:[<ffffffff810ba1c8>] [<ffffffff810ba1c8>] __bfs+0xc8/0x260 [ 5954.096665] RSP: 0018:ffff8801175539a8 EFLAGS: 00010046 [ 5954.096665] RAX: ffffffff8267d980 RBX: ffffffff8267d980 RCX: ffff880117553a48 [ 5954.096665] RDX: ffff8801175539d0 RSI: 0000000000000000 RDI: ffff880117553a48 [ 5954.096665] RBP: ffff880117553a08 R08: 0000000000000000 R09: 0000000000000000 [ 5954.096665] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000000 [ 5954.096665] R13: ffffffff810b8e20 R14: ffff880117553a90 R15: 0000000000000000 [ 5954.096665] FS: 00007f4594cf3700(0000) GS:ffff8800dfa00000(0000) knlGS:0000000000000000 [ 5954.096665] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 5954.096665] CR2: 00007f2f21e89c60 CR3: 0000000110b0f000 CR4: 00000000000006e0 [ 5954.096665] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 5954.096665] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [ 5954.096665] Process rm (pid: 2927, threadinfo ffff880117552000, task ffff88010ff10b00) [ 5954.096665] Stack: [ 5954.096665] ffffffff8267d868 00007fffa02d1428 ffff8800ffffffff ffff880100000000 [ 5954.096665] 000000000000b720 ffff880117553a48 ffffffff8267d868 ffff880117553a48 [ 5954.096665] 0000000000000000 ffff88010ff10b00 0000000000000000 ffffffff81dacba0 [ 5954.096665] Call Trace: [ 5954.096665] [<ffffffff810ba4d2>] find_usage_backwards+0x42/0x80 [ 5954.096665] [<ffffffff810bcec4>] check_usage_backwards+0x64/0xf0 [ 5954.096665] [<ffffffff8104796f>] ? save_stack_trace+0x2f/0x50 [ 5954.096665] [<ffffffff810bce60>] ? check_usage_backwards+0x0/0xf0 [ 5954.096665] [<ffffffff810bd9a9>] mark_lock+0x1a9/0x440 [ 5954.096665] [<ffffffff810be989>] __lock_acquire+0x5a9/0x14b0 [ 5954.096665] [<ffffffff810be716>] ? __lock_acquire+0x336/0x14b0 [ 5954.096665] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0 [ 5954.096665] [<ffffffff810bf944>] lock_acquire+0xb4/0x140 [ 5954.096665] [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430 [ 5954.096665] [<ffffffff81b19d86>] _raw_spin_lock+0x36/0x70 [ 5954.096665] [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430 [ 5954.096665] [<ffffffff81177a1c>] shrink_dentry_list+0x5c/0x430 [ 5954.096665] [<ffffffff811779c0>] ? shrink_dentry_list+0x0/0x430 [ 5954.096665] [<ffffffff816b9c7e>] ? do_raw_spin_unlock+0x5e/0xb0 [ 5954.096665] [<ffffffff81177f2d>] __shrink_dcache_sb+0x13d/0x1c0 [ 5954.096665] [<ffffffff811784bf>] shrink_dcache_parent+0x32f/0x390 [ 5954.096665] [<ffffffff8116d31d>] dentry_unhash+0x3d/0x70 [ 5954.096665] [<ffffffff8116d3b0>] vfs_rmdir+0x60/0xe0 [ 5954.096665] [<ffffffff8116f673>] do_rmdir+0x113/0x130 [ 5954.096665] [<ffffffff8103a03a>] ? sysret_check+0x2e/0x69
Quoted 8 lines
[ 5954.096665] [<ffffffff81b19ae9>] ? trace_hardirqs_on_thunk+0x3a/0x3f [ 5954.096665] [<ffffffff8116f6c5>] sys_unlinkat+0x35/0x40 [ 5954.096665] [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b [ 5954.096665] Code: 0a 89 05 dc 0f a8 01 48 8b 41 10 48 85 c0 0f 84 1f 01 00 00 48 8d 98 70 01 00 00 48 05 80 01 00 00 45 85 c0 48 0f 44 d8 4c 8b 3b <49> 8b 07 49 39 [ 5954.096665] RIP [<ffffffff810ba1c8>] __bfs+0xc8/0x260 [ 5954.096665] RSP <ffff8801175539a8> [ 5954.096665] CR2: 0000000000000000 [ 5954.127991] ---[ end trace 85a6727c2d4e3d90 ]---
So I vfs-scale-working branch may not be entirely in the clear, seeing as it touches
the code lower in the call chain. However I don't know what can cause lockdep to go off
the rails like this.
There is a sequence I used to hack around lockdep nesting restrictions, following this
pattern:
repeat:
spin_lock(&parent->d_lock);
spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
/* do stuff */
spin_unlock(&parent->d_lock);
spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
parent = dentry;
spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
goto repeat;It's not directly in this call chain, but I wonder if it could have caused any problem?
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 10 lines · Nick Piggin
repeat:
spin_lock(&parent->d_lock); spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); /* do stuff */ spin_unlock(&parent->d_lock); spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_); parent = dentry; spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_); goto repeat;
shouldn't that be s/this_parent/parent/ ?
So what you're trying to do is:
A -> B -> C -> ...
lock A
lock B, nested
unlock A
flip B from nested to top
lock C, nested
unlock B
flip C from nested to top
lock ...
Anyway, the way to write that is something like:
lock_set_subclass(&detry->d_lock.dep_map, 0, _RET_IP_);
Which will reset the subclass of the held lock from DENTRY_D_LOCK_NESTED to 0.
This is also used in double_unlock_balance(), we go into double_lock_balance() with this_rq locked and want to lock busiest, because of the lock ordering we might need to unlock this_rq and lock busiest first, at which point this_rq is nested.
On unlock we thus need to map this_rq back to subclass 0 (which it had before double_lock_balance(), because otherwise subsequent lock operations will be done against the subclass and confuse things.
Nick Piggin
reviewerQuoted 12 lines · Peter Zijlstra
Quoted 10 lines · Nick Piggin
repeat:
spin_lock(&parent->d_lock);spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); /* do stuff */
spin_unlock(&parent->d_lock);spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_); parent = dentry;spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_); goto repeat;
Quoted 1 line
shouldn't that be s/this_parent/parent/ ?
Yes, typo in my pseudo code.
Quoted 13 lines
So what you're trying to do is:
A -> B -> C -> ...
lock A
lock B, nested
unlock A
flip B from nested to top
lock C, nested
unlock B
flip C from nested to top
lock ...
Quoted 6 lines
Anyway, the way to write that is something like:
lock_set_subclass(&detry->d_lock.dep_map, 0, _RET_IP_);
Which will reset the subclass of the held lock from
DENTRY_D_LOCK_NESTEDto 0.
OK, thanks. My version should not have caused any problems though, right? Any idea what might have caused Dave's crash?
Quoted 8 lines
This is also used in
double_unlock_balance(), we go intodouble_lock_balance()withthis_rqlocked and want to lock busiest, because of the lock ordering we might need to unlockthis_rqand lock busiest first, at which pointthis_rqis nested.On unlock we thus need to map
this_rqback to subclass 0 (which it had beforedouble_lock_balance(), because otherwise subsequent lock operations will be done against the subclass and confuse things.
Thanks,
Nick
Quoted 2 lines · Nick Piggin
OK, thanks. My version should not have caused any problems though, right?
I tihnk so, yes, altough looking at it again I wonder why you use spin_aquire(.trylock=1) -- but that too shouldn't cause anything like the explosion.
Quoted 1 line
Any idea what might have caused Dave's crash?
Not directly, no. Usually lockdep crashes indicate use after free like things, where we try to lock a lock that's been scribbled on. But that usually explodes a bit earlier.
You faulting in the middle of that breath-first-search does suggest some data corruption, but I'm not quite sure what kind, I can't remember ever having seem something like this before.
I've CC'ed Ming Lei who wrote the bfs search, maybe he's got an idea.
Copy of the splat:
---
Quoted 49 lines
[ 5954.061633] BUG: sleeping function called from invalid context at arch/x86/mm/fault.c:1081
[ 5954.062466] in_atomic(): 0, irqs_disabled(): 1, pid: 2927, name: rm
[ 5954.063122] 3 locks held by rm/2927:
[ 5954.063476] #0: (&sb->s_type->i_mutex_key#12/1){+.+.+.}, at: [<ffffffff8116f5e1>] do_rmdir+0x81/0x130
[ 5954.064014] #1: (&sb->s_type->i_mutex_key#12){+.+.+.}, at: [<ffffffff8116d3a8>] vfs_rmdir+0x58/0xe0
[ 5954.064014] #2: (rcu_read_lock){.+.+..}, at: [<ffffffff811779c0>] shrink_dentry_list+0x0/0x430
[ 5954.064014] irq event stamp: 1484376719
[ 5954.064014] hardirqs last enabled at (1484376719): [<ffffffff810ebf07>] __call_rcu+0xd7/0x1a0
[ 5954.064014] hardirqs last disabled at (1484376718): [<ffffffff810ebe7a>] __call_rcu+0x4a/0x1a0
[ 5954.064014] softirqs last enabled at (1484376586): [<ffffffff8108b911>] __do_softirq+0x161/0x270
[ 5954.064014] softirqs last disabled at (1484376581): [<ffffffff8103af1c>] call_softirq+0x1c/0x50
[ 5954.064014] Pid: 2927, comm: rm Not tainted 2.6.37-rc4-dgc+ #794
[ 5954.064014] Call Trace:
[ 5954.064014] [<ffffffff810b95b0>] ? print_irqtrace_events+0xd0/0xe0
[ 5954.064014] [<ffffffff81076455>] __might_sleep+0xf5/0x130
[ 5954.064014] [<ffffffff81b1e603>] do_page_fault+0x103/0x4f0
[ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014] [<ffffffff81b19b28>] ? trace_hardirqs_off_thunk+0x3a/0x3c
[ 5954.064014] [<ffffffff810b8e20>] ? usage_match+0x0/0x20
[ 5954.064014] [<ffffffff81b1af25>] page_fault+0x25/0x30
[ 5954.064014] [<ffffffff810b8e20>] ? usage_match+0x0/0x20
[ 5954.064014] [<ffffffff810ba1c8>] ? __bfs+0xc8/0x260
[ 5954.064014] [<ffffffff810ba123>] ? __bfs+0x23/0x260
[ 5954.064014] [<ffffffff810ba4d2>] find_usage_backwards+0x42/0x80
[ 5954.064014] [<ffffffff810bcec4>] check_usage_backwards+0x64/0xf0
[ 5954.064014] [<ffffffff8104796f>] ? save_stack_trace+0x2f/0x50
[ 5954.064014] [<ffffffff810bce60>] ? check_usage_backwards+0x0/0xf0
[ 5954.064014] [<ffffffff810bd9a9>] mark_lock+0x1a9/0x440
[ 5954.064014] [<ffffffff810be989>] __lock_acquire+0x5a9/0x14b0
[ 5954.064014] [<ffffffff810be716>] ? __lock_acquire+0x336/0x14b0
[ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0
[ 5954.064014] [<ffffffff810bf944>] lock_acquire+0xb4/0x140
[ 5954.064014] [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.064014] [<ffffffff81b19d86>] _raw_spin_lock+0x36/0x70
[ 5954.064014] [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430
[ 5954.064014] [<ffffffff81177a1c>] shrink_dentry_list+0x5c/0x430
[ 5954.064014] [<ffffffff811779c0>] ? shrink_dentry_list+0x0/0x430
[ 5954.064014] [<ffffffff816b9c7e>] ? do_raw_spin_unlock+0x5e/0xb0
[ 5954.064014] [<ffffffff81177f2d>] __shrink_dcache_sb+0x13d/0x1c0
[ 5954.064014] [<ffffffff811784bf>] shrink_dcache_parent+0x32f/0x390
[ 5954.064014] [<ffffffff8116d31d>] dentry_unhash+0x3d/0x70
[ 5954.064014] [<ffffffff8116d3b0>] vfs_rmdir+0x60/0xe0
[ 5954.064014] [<ffffffff8116f673>] do_rmdir+0x113/0x130
[ 5954.064014] [<ffffffff8103a03a>] ? sysret_check+0x2e/0x69
[ 5954.064014] [<ffffffff81b19ae9>] ? trace_hardirqs_on_thunk+0x3a/0x3f
[ 5954.064014] [<ffffffff8116f6c5>] sys_unlinkat+0x35/0x40
[ 5954.064014] [<ffffffff8103a002>] system_call_fastpath+0x16/0x1bQuoted 110 lines · Nick Piggin
Quoted 41 lines · Dave Chinner
[ 5954.061633] BUG: sleeping function called from invalid context at arch/x86/mm/fault.c:1081 [ 5954.062466] in_atomic(): 0, irqs_disabled(): 1, pid: 2927, name: rm [ 5954.063122] 3 locks held by rm/2927: [ 5954.063476] #0: (&sb->s_type->i_mutex_key#12/1){+.+.+.}, at: [<ffffffff8116f5e1>] do_rmdir+0x81/0x130 [ 5954.064014] #1: (&sb->s_type->i_mutex_key#12){+.+.+.}, at: [<ffffffff8116d3a8>] vfs_rmdir+0x58/0xe0 [ 5954.064014] #2: (rcu_read_lock){.+.+..}, at: [<ffffffff811779c0>] shrink_dentry_list+0x0/0x430 [ 5954.064014] irq event stamp: 1484376719 [ 5954.064014] hardirqs last enabled at (1484376719): [<ffffffff810ebf07>] __call_rcu+0xd7/0x1a0 [ 5954.064014] hardirqs last disabled at (1484376718): [<ffffffff810ebe7a>] __call_rcu+0x4a/0x1a0 [ 5954.064014] softirqs last enabled at (1484376586): [<ffffffff8108b911>] __do_softirq+0x161/0x270 [ 5954.064014] softirqs last disabled at (1484376581): [<ffffffff8103af1c>] call_softirq+0x1c/0x50 [ 5954.064014] Pid: 2927, comm: rm Not tainted 2.6.37-rc4-dgc+ #794 [ 5954.064014] Call Trace: [ 5954.064014] [<ffffffff810b95b0>] ? print_irqtrace_events+0xd0/0xe0 [ 5954.064014] [<ffffffff81076455>] __might_sleep+0xf5/0x130 [ 5954.064014] [<ffffffff81b1e603>] do_page_fault+0x103/0x4f0 [ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0 [ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0 [ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0 [ 5954.064014] [<ffffffff81b19b28>] ? trace_hardirqs_off_thunk+0x3a/0x3c [ 5954.064014] [<ffffffff810b8e20>] ? usage_match+0x0/0x20 [ 5954.064014] [<ffffffff81b1af25>] page_fault+0x25/0x30 [ 5954.064014] [<ffffffff810b8e20>] ? usage_match+0x0/0x20 [ 5954.064014] [<ffffffff810ba1c8>] ? __bfs+0xc8/0x260 [ 5954.064014] [<ffffffff810ba123>] ? __bfs+0x23/0x260 [ 5954.064014] [<ffffffff810ba4d2>] find_usage_backwards+0x42/0x80 [ 5954.064014] [<ffffffff810bcec4>] check_usage_backwards+0x64/0xf0 [ 5954.064014] [<ffffffff8104796f>] ? save_stack_trace+0x2f/0x50 [ 5954.064014] [<ffffffff810bce60>] ? check_usage_backwards+0x0/0xf0 [ 5954.064014] [<ffffffff810bd9a9>] mark_lock+0x1a9/0x440 [ 5954.064014] [<ffffffff810be989>] __lock_acquire+0x5a9/0x14b0 [ 5954.064014] [<ffffffff810be716>] ? __lock_acquire+0x336/0x14b0 [ 5954.064014] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0 [ 5954.064014] [<ffffffff810bf944>] lock_acquire+0xb4/0x140 [ 5954.064014] [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430 [ 5954.064014] [<ffffffff81b19d86>] _raw_spin_lock+0x36/0x70 [ 5954.064014] [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430 [ 5954.064014] [<ffffffff81177a1c>] shrink_dentry_list+0x5c/0x430 [ 5954.064014] [<ffffffff811779c0>] ? shrink_dentry_list+0x0/0x430 [ 5954.064014] [<ffffffff816b9c7e>] ? do_raw_spin_unlock+0x5e/0xb0 [ 5954.064014] [<ffffffff81177f2d>] __shrink_dcache_sb+0x13d/0x1c0Quoted 8 lines
[ 5954.064014] [<ffffffff811784bf>] shrink_dcache_parent+0x32f/0x390 [ 5954.064014] [<ffffffff8116d31d>] dentry_unhash+0x3d/0x70 [ 5954.064014] [<ffffffff8116d3b0>] vfs_rmdir+0x60/0xe0 [ 5954.064014] [<ffffffff8116f673>] do_rmdir+0x113/0x130 [ 5954.064014] [<ffffffff8103a03a>] ? sysret_check+0x2e/0x69 [ 5954.064014] [<ffffffff81b19ae9>] ? trace_hardirqs_on_thunk+0x3a/0x3f [ 5954.064014] [<ffffffff8116f6c5>] sys_unlinkat+0x35/0x40 [ 5954.064014] [<ffffffff8103a002>] system_call_fastpath+0x16/0x1bSeems that lockdep exploded.
Quoted 48 lines
[ 5954.092916] BUG: unable to handle kernel NULL pointer dereference at (null) [ 5954.093806] IP: [<ffffffff810ba1c8>] __bfs+0xc8/0x260 [ 5954.094331] PGD 1084e5067 PUD 102368067 PMD 0 [ 5954.094830] Oops: 0000 [#1] SMP [ 5954.095194] last sysfs file: /sys/devices/system/cpu/online [ 5954.095760] CPU 6 [ 5954.095954] Modules linked in: [ 5954.096319] [ 5954.096483] Pid: 2927, comm: rm Not tainted 2.6.37-rc4-dgc+ #794 /Bochs [ 5954.096665] RIP: 0010:[<ffffffff810ba1c8>] [<ffffffff810ba1c8>] __bfs+0xc8/0x260 [ 5954.096665] RSP: 0018:ffff8801175539a8 EFLAGS: 00010046 [ 5954.096665] RAX: ffffffff8267d980 RBX: ffffffff8267d980 RCX: ffff880117553a48 [ 5954.096665] RDX: ffff8801175539d0 RSI: 0000000000000000 RDI: ffff880117553a48 [ 5954.096665] RBP: ffff880117553a08 R08: 0000000000000000 R09: 0000000000000000 [ 5954.096665] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000000 [ 5954.096665] R13: ffffffff810b8e20 R14: ffff880117553a90 R15: 0000000000000000 [ 5954.096665] FS: 00007f4594cf3700(0000) GS:ffff8800dfa00000(0000) knlGS:0000000000000000 [ 5954.096665] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 5954.096665] CR2: 00007f2f21e89c60 CR3: 0000000110b0f000 CR4: 00000000000006e0 [ 5954.096665] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 5954.096665] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [ 5954.096665] Process rm (pid: 2927, threadinfo ffff880117552000, task ffff88010ff10b00) [ 5954.096665] Stack: [ 5954.096665] ffffffff8267d868 00007fffa02d1428 ffff8800ffffffff ffff880100000000 [ 5954.096665] 000000000000b720 ffff880117553a48 ffffffff8267d868 ffff880117553a48 [ 5954.096665] 0000000000000000 ffff88010ff10b00 0000000000000000 ffffffff81dacba0 [ 5954.096665] Call Trace: [ 5954.096665] [<ffffffff810ba4d2>] find_usage_backwards+0x42/0x80 [ 5954.096665] [<ffffffff810bcec4>] check_usage_backwards+0x64/0xf0 [ 5954.096665] [<ffffffff8104796f>] ? save_stack_trace+0x2f/0x50 [ 5954.096665] [<ffffffff810bce60>] ? check_usage_backwards+0x0/0xf0 [ 5954.096665] [<ffffffff810bd9a9>] mark_lock+0x1a9/0x440 [ 5954.096665] [<ffffffff810be989>] __lock_acquire+0x5a9/0x14b0 [ 5954.096665] [<ffffffff810be716>] ? __lock_acquire+0x336/0x14b0 [ 5954.096665] [<ffffffff810645b8>] ? pvclock_clocksource_read+0x58/0xd0 [ 5954.096665] [<ffffffff810bf944>] lock_acquire+0xb4/0x140 [ 5954.096665] [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430 [ 5954.096665] [<ffffffff81b19d86>] _raw_spin_lock+0x36/0x70 [ 5954.096665] [<ffffffff81177a1c>] ? shrink_dentry_list+0x5c/0x430 [ 5954.096665] [<ffffffff81177a1c>] shrink_dentry_list+0x5c/0x430 [ 5954.096665] [<ffffffff811779c0>] ? shrink_dentry_list+0x0/0x430 [ 5954.096665] [<ffffffff816b9c7e>] ? do_raw_spin_unlock+0x5e/0xb0 [ 5954.096665] [<ffffffff81177f2d>] __shrink_dcache_sb+0x13d/0x1c0 [ 5954.096665] [<ffffffff811784bf>] shrink_dcache_parent+0x32f/0x390 [ 5954.096665] [<ffffffff8116d31d>] dentry_unhash+0x3d/0x70 [ 5954.096665] [<ffffffff8116d3b0>] vfs_rmdir+0x60/0xe0 [ 5954.096665] [<ffffffff8116f673>] do_rmdir+0x113/0x130 [ 5954.096665] [<ffffffff8103a03a>] ? sysret_check+0x2e/0x69Quoted 8 lines
[ 5954.096665] [<ffffffff81b19ae9>] ? trace_hardirqs_on_thunk+0x3a/0x3f [ 5954.096665] [<ffffffff8116f6c5>] sys_unlinkat+0x35/0x40 [ 5954.096665] [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b [ 5954.096665] Code: 0a 89 05 dc 0f a8 01 48 8b 41 10 48 85 c0 0f 84 1f 01 00 00 48 8d 98 70 01 00 00 48 05 80 01 00 00 45 85 c0 48 0f 44 d8 4c 8b 3b <49> 8b 07 49 39 [ 5954.096665] RIP [<ffffffff810ba1c8>] __bfs+0xc8/0x260 [ 5954.096665] RSP <ffff8801175539a8> [ 5954.096665] CR2: 0000000000000000 [ 5954.127991] ---[ end trace 85a6727c2d4e3d90 ]---
Quoted 1 line
So vfs-scale-working branch may not be entirely in the clear, seeing
Ah, may have been a stupid little bug. The list entry check was being done and then the pointer reloaded to be used.
What does the asm for shrink_dentry_list look like (before this patch)?
Thanks,
Nick
fs/dcache.c +4 -4
@@ -657,10 +657,10 @@ static void shrink_dentry_list(struct listruct dentry *dentry;rcu_read_lock (); -while (!list_empty (list)) { - dentry = list_entry(list->prev, struct dentry, d_lru); - -/* Don't need RCU dereference because we recheck under lock */ +for (;;) { + dentry = list_entry_rcu(list->prev, struct dentry, d_lru); +if (&dentry->d_lru == list) +break ;spin_lock (&dentry->d_lock); if (dentry != list_entry(list->prev, struct dentry, d_lru)) {spin_unlock (&dentry->d_lock);
Quoted 11 lines · Nick Piggin
git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.gitvfs-scale-workingHere is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:
* preparation patches
* introduce new locks to take overdcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking steps
Stress test doing:
single thread 50M inode create
single thread rm -rf
2-way 50M inode create
2-way rm -rf
4-way 50M inode create
4-way rm -rf
8-way 50M inode create
8-way rm -rf
8-way 250M inode create
8-way rm -rf
Failed about 5 minutes into the "4-way rm -rf" (~3 hours into the test) with a CPU stuck spinning on here:
[37372.084012] NMI backtrace for cpu 5 [37372.084012] CPU 5 [37372.084012] Modules linked in: [37372.084012] [37372.084012] Pid: 15214, comm: rm Not tainted 2.6.37-rc4-dgc+ #797 /Bochs [37372.084012] RIP: 0010:[<ffffffff810643c4>] [<ffffffff810643c4>] __ticket_spin_lock+0x14/0x20 [37372.084012] RSP: 0018:ffff880114643c98 EFLAGS: 00000213 [37372.084012] RAX: 0000000000008801 RBX: ffff8800687be6c0 RCX: ffff8800c4eb2688 [37372.084012] RDX: ffff880114643d38 RSI: ffff8800dfd4ea80 RDI: ffff880114643d14 [37372.084012] RBP: ffff880114643c98 R08: 0000000000000003 R09: 0000000000000000 [37372.084012] R10: 0000000000000000 R11: dead000000200200 R12: ffff880114643d14 [37372.084012] R13: ffff880114643cb8 R14: ffff880114643d38 R15: ffff8800687be71c [37372.084012] FS: 00007fd6d7c93700(0000) GS:ffff8800dfd40000(0000) knlGS:0000000000000000 [37372.084012] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [37372.084012] CR2: 0000000000bbd108 CR3: 0000000107146000 CR4: 00000000000006e0 [37372.084012] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [37372.084012] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [37372.084012] Process rm (pid: 15214, threadinfo ffff880114642000, task ffff88011b16f890) [37372.084012] Stack: [37372.084012] ffff880114643ca8 ffffffff81ad044e ffff880114643cf8 ffffffff81167ae7 [37372.084012] 0000000000000000 ffff880114643d38 000000000000000e ffff88011901d800 [37372.084012] ffff8800cdb7cf5c ffff88011901d8e0 0000000000000000 0000000000000000 [37372.084012] Call Trace: [37372.084012] [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370 [37372.084012] [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0 [37372.084012] [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0 [37372.084012] [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff8115daa2>] dentry_unhash+0x42/0x80 [37372.084012] [<ffffffff8115db48>] vfs_rmdir+0x68/0x100 [37372.084012] [<ffffffff8115fd93>] do_rmdir+0x113/0x130 [37372.084012] [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90 [37372.084012] [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40 [37372.084012] [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b [37372.084012] Code: c1 c1 41 06 81 e9 dd fe ff ff 90 90 90 90 90 90 90 90 90 90 90 90 90 55 b8 00 01 00 00 48 89 e5 f0 66 0f c1 07 38 e0 74 06 f3 90 <8a> 07 eb f6 c9 c3 66 0f 1f 44 00 00 55 48 89 e5 0f b7 07 38 e0 [37372.084012] Call Trace: [37372.084012] [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370 [37372.084012] [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0 [37372.084012] [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0 [37372.084012] [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff8115daa2>] dentry_unhash+0x42/0x80 [37372.084012] [<ffffffff8115db48>] vfs_rmdir+0x68/0x100 [37372.084012] [<ffffffff8115fd93>] do_rmdir+0x113/0x130 [37372.084012] [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90 [37372.084012] [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40 [37372.084012] [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
Nick Piggin
reviewerQuoted 55 lines · Dave Chinner
Quoted 11 lines · Nick Piggin
git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.gitvfs-scale-workingHere is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:
* preparation patches
* introduce new locks to take overdcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking stepsStress test doing:
single thread 50M inode create
single thread rm -rf
2-way 50M inode create
2-way rm -rf
4-way 50M inode create
4-way rm -rf
8-way 50M inode create
8-way rm -rf
8-way 250M inode create
8-way rm -rfFailed about 5 minutes into the "4-way rm -rf" (~3 hours into the test) with a CPU stuck spinning on here:
[37372.084012] NMI backtrace for cpu 5 [37372.084012] CPU 5 [37372.084012] Modules linked in: [37372.084012] [37372.084012] Pid: 15214, comm: rm Not tainted 2.6.37-rc4-dgc+ #797 /Bochs [37372.084012] RIP: 0010:[<ffffffff810643c4>] [<ffffffff810643c4>] __ticket_spin_lock+0x14/0x20 [37372.084012] RSP: 0018:ffff880114643c98 EFLAGS: 00000213 [37372.084012] RAX: 0000000000008801 RBX: ffff8800687be6c0 RCX: ffff8800c4eb2688 [37372.084012] RDX: ffff880114643d38 RSI: ffff8800dfd4ea80 RDI: ffff880114643d14 [37372.084012] RBP: ffff880114643c98 R08: 0000000000000003 R09: 0000000000000000 [37372.084012] R10: 0000000000000000 R11: dead000000200200 R12: ffff880114643d14 [37372.084012] R13: ffff880114643cb8 R14: ffff880114643d38 R15: ffff8800687be71c [37372.084012] FS: 00007fd6d7c93700(0000) GS:ffff8800dfd40000(0000) knlGS:0000000000000000 [37372.084012] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [37372.084012] CR2: 0000000000bbd108 CR3: 0000000107146000 CR4: 00000000000006e0 [37372.084012] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [37372.084012] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [37372.084012] Process rm (pid: 15214, threadinfo ffff880114642000, task ffff88011b16f890) [37372.084012] Stack: [37372.084012] ffff880114643ca8 ffffffff81ad044e ffff880114643cf8 ffffffff81167ae7 [37372.084012] 0000000000000000 ffff880114643d38 000000000000000e ffff88011901d800 [37372.084012] ffff8800cdb7cf5c ffff88011901d8e0 0000000000000000 0000000000000000 [37372.084012] Call Trace: [37372.084012] [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370 [37372.084012] [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0
Quoted 8 lines
[37372.084012] [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0 [37372.084012] [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff8115daa2>] dentry_unhash+0x42/0x80 [37372.084012] [<ffffffff8115db48>] vfs_rmdir+0x68/0x100 [37372.084012] [<ffffffff8115fd93>] do_rmdir+0x113/0x130 [37372.084012] [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90 [37372.084012] [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40 [37372.084012] [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
OK good, with any luck, that's the same bug.
Is this XFS? Is there any concurrent activity happening on the same dentries? Ie. are the rm -rf threads running on the same directories, or is there any reclaim
happening in the background?
Thanks,
Nick
Quoted 65 lines · Nick Piggin
Quoted 55 lines · Dave Chinner
Quoted 11 lines · Nick Piggin
git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.gitvfs-scale-workingHere is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:
* preparation patches
* introduce new locks to take overdcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking stepsStress test doing:
single thread 50M inode create
single thread rm -rf
2-way 50M inode create
2-way rm -rf
4-way 50M inode create
4-way rm -rf
8-way 50M inode create
8-way rm -rf
8-way 250M inode create
8-way rm -rfFailed about 5 minutes into the "4-way rm -rf" (~3 hours into the test) with a CPU stuck spinning on here:
[37372.084012] NMI backtrace for cpu 5 [37372.084012] CPU 5 [37372.084012] Modules linked in: [37372.084012] [37372.084012] Pid: 15214, comm: rm Not tainted 2.6.37-rc4-dgc+ #797 /Bochs [37372.084012] RIP: 0010:[<ffffffff810643c4>] [<ffffffff810643c4>] __ticket_spin_lock+0x14/0x20 [37372.084012] RSP: 0018:ffff880114643c98 EFLAGS: 00000213 [37372.084012] RAX: 0000000000008801 RBX: ffff8800687be6c0 RCX: ffff8800c4eb2688 [37372.084012] RDX: ffff880114643d38 RSI: ffff8800dfd4ea80 RDI: ffff880114643d14 [37372.084012] RBP: ffff880114643c98 R08: 0000000000000003 R09: 0000000000000000 [37372.084012] R10: 0000000000000000 R11: dead000000200200 R12: ffff880114643d14 [37372.084012] R13: ffff880114643cb8 R14: ffff880114643d38 R15: ffff8800687be71c [37372.084012] FS: 00007fd6d7c93700(0000) GS:ffff8800dfd40000(0000) knlGS:0000000000000000 [37372.084012] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [37372.084012] CR2: 0000000000bbd108 CR3: 0000000107146000 CR4: 00000000000006e0 [37372.084012] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [37372.084012] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [37372.084012] Process rm (pid: 15214, threadinfo ffff880114642000, task ffff88011b16f890) [37372.084012] Stack: [37372.084012] ffff880114643ca8 ffffffff81ad044e ffff880114643cf8 ffffffff81167ae7 [37372.084012] 0000000000000000 ffff880114643d38 000000000000000e ffff88011901d800 [37372.084012] ffff8800cdb7cf5c ffff88011901d8e0 0000000000000000 0000000000000000 [37372.084012] Call Trace: [37372.084012] [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370 [37372.084012] [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0Quoted 8 lines
[37372.084012] [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0 [37372.084012] [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff8115daa2>] dentry_unhash+0x42/0x80 [37372.084012] [<ffffffff8115db48>] vfs_rmdir+0x68/0x100 [37372.084012] [<ffffffff8115fd93>] do_rmdir+0x113/0x130 [37372.084012] [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90 [37372.084012] [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40 [37372.084012] [<ffffffff8103a002>] system_call_fastpath+0x16/0x1b
Quoted 3 lines
OK good, with any luck, that's the same bug.
Is this XFS?
Yes.
Quoted 1 line
Is there any concurrent activity happening on the same dentries?
Not from an application perspective.
Quoted 1 line
Ie. are the rm -rf threads running on the same directories,
No, each thread operating on a different directory.
Quoted 1 line
or is there any reclaim happening in the background?
IIRC, kswapd was consuming about 5-10% of a CPU during parallel unlink tests. Mainly reclaiming XFS inodes, I think, but there may be dentry cache reclaim going as well.
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
Quoted 78 lines · Dave Chinner
Quoted 65 lines · Nick Piggin
Quoted 55 lines · Dave Chinner
Quoted 11 lines · Nick Piggin
git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.gitvfs-scale-workingHere is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:
* preparation patches
* introduce new locks to take overdcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking stepsStress test doing:
single thread 50M inode create
single thread rm -rf
2-way 50M inode create
2-way rm -rf
4-way 50M inode create
4-way rm -rf
8-way 50M inode create
8-way rm -rf
8-way 250M inode create
8-way rm -rfFailed about 5 minutes into the "4-way rm -rf" (~3 hours into the test) with a CPU stuck spinning on here:
[37372.084012] NMI backtrace for cpu 5 [37372.084012] CPU 5 [37372.084012] Modules linked in: [37372.084012] [37372.084012] Pid: 15214, comm: rm Not tainted 2.6.37-rc4-dgc+ #797 /Bochs [37372.084012] RIP: 0010:[<ffffffff810643c4>] [<ffffffff810643c4>] __ticket_spin_lock+0x14/0x20 [37372.084012] RSP: 0018:ffff880114643c98 EFLAGS: 00000213 [37372.084012] RAX: 0000000000008801 RBX: ffff8800687be6c0 RCX: ffff8800c4eb2688 [37372.084012] RDX: ffff880114643d38 RSI: ffff8800dfd4ea80 RDI: ffff880114643d14 [37372.084012] RBP: ffff880114643c98 R08: 0000000000000003 R09: 0000000000000000 [37372.084012] R10: 0000000000000000 R11: dead000000200200 R12: ffff880114643d14 [37372.084012] R13: ffff880114643cb8 R14: ffff880114643d38 R15: ffff8800687be71c [37372.084012] FS: 00007fd6d7c93700(0000) GS:ffff8800dfd40000(0000) knlGS:0000000000000000 [37372.084012] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [37372.084012] CR2: 0000000000bbd108 CR3: 0000000107146000 CR4: 00000000000006e0 [37372.084012] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [37372.084012] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [37372.084012] Process rm (pid: 15214, threadinfo ffff880114642000, task ffff88011b16f890) [37372.084012] Stack: [37372.084012] ffff880114643ca8 ffffffff81ad044e ffff880114643cf8 ffffffff81167ae7 [37372.084012] 0000000000000000 ffff880114643d38 000000000000000e ffff88011901d800 [37372.084012] ffff8800cdb7cf5c ffff88011901d8e0 0000000000000000 0000000000000000 [37372.084012] Call Trace: [37372.084012] [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370 [37372.084012] [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0Quoted 8 lines
[37372.084012] [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0 [37372.084012] [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff8115daa2>] dentry_unhash+0x42/0x80 [37372.084012] [<ffffffff8115db48>] vfs_rmdir+0x68/0x100 [37372.084012] [<ffffffff8115fd93>] do_rmdir+0x113/0x130 [37372.084012] [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90 [37372.084012] [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40 [37372.084012] [<ffffffff8103a002>] system_call_fastpath+0x16/0x1bQuoted 3 lines
OK good, with any luck, that's the same bug.
Is this XFS?
Yes.
Quoted 1 line
Is there any concurrent activity happening on the same dentries?
Not from an application perspective.
Quoted 1 line
Ie. are the rm -rf threads running on the same directories,
Quoted 7 lines
No, each thread operating on a different directory.
Quoted 1 line
or is there any reclaim happening in the background?
IIRC, kswapd was consuming about 5-10% of a CPU during parallel unlink tests. Mainly reclaiming XFS inodes, I think, but there may be dentry cache reclaim going as well.
Turns out that the kswapd peaks are upwards of 50% of a CPU for a few seconds, then idle for 10-15s. Typical perf top output of kswapd while it is active during unlinks is:
samples pcnt function DSO
_______ _____ ___________________________ _________________
17168.00 10.2% __call_rcu [kernel.kallsyms]
13223.00 7.8% kmem_cache_free [kernel.kallsyms]
12917.00 7.6% down_write [kernel.kallsyms]
12665.00 7.5% xfs_iunlock [kernel.kallsyms]
10493.00 6.2% xfs_reclaim_inode_grab [kernel.kallsyms]
9314.00 5.5% __lookup_tag [kernel.kallsyms]
9040.00 5.4% radix_tree_delete [kernel.kallsyms]
8694.00 5.1% is_bad_inode [kernel.kallsyms]
7639.00 4.5% __ticket_spin_lock [kernel.kallsyms]
6821.00 4.0% _raw_spin_unlock_irqrestore [kernel.kallsyms]
5484.00 3.2% __d_drop [kernel.kallsyms]
5114.00 3.0% xfs_reclaim_inode [kernel.kallsyms]
4626.00 2.7% __rcu_process_callbacks [kernel.kallsyms]
3556.00 2.1% up_write [kernel.kallsyms]
3206.00 1.9% _cond_resched [kernel.kallsyms]
3129.00 1.9% xfs_qm_dqdetach [kernel.kallsyms]
2327.00 1.4% radix_tree_tag_clear [kernel.kallsyms]
2327.00 1.4% call_rcu_sched [kernel.kallsyms]
2262.00 1.3% __ticket_spin_unlock [kernel.kallsyms]
2215.00 1.3% xfs_ilock [kernel.kallsyms]
2200.00 1.3% radix_tree_gang_lookup_tag [kernel.kallsyms]
1982.00 1.2% xfs_reclaim_inodes_ag [kernel.kallsyms]
1736.00 1.0% xfs_trans_unlocked_item [kernel.kallsyms]
1707.00 1.0% __ticket_spin_trylock [kernel.kallsyms]
1688.00 1.0% xfs_perag_get_tag [kernel.kallsyms]
1660.00 1.0% flat_send_IPI_mask [kernel.kallsyms]
1538.00 0.9% xfs_inode_item_destroy [kernel.kallsyms]
1312.00 0.8% __shrink_dcache_sb [kernel.kallsyms]
940.00 0.6% xfs_perag_put [kernel.kallsyms]So there is some dentry cache reclaim going on.
FWIW, it appears there is quite a lot of RCU freeing overhead (~15% more CPU time) in the work kswapd is doing during these unlinks, too. I just had a look at kswapd when a 8-way create is running - it's running at 50-60% of a cpu for seconds at a time. I caught this while it was doing pure XFS inode cache reclaim (~10s sample, kswapd reclaimed ~1M inodes):
samples pcnt function DSO
_______ _____ ___________________________ _________________
27171.00 9.0% __call_rcu [kernel.kallsyms]
21491.00 7.1% down_write [kernel.kallsyms]
20916.00 6.9% xfs_reclaim_inode [kernel.kallsyms]
20313.00 6.7% radix_tree_delete [kernel.kallsyms]
15828.00 5.3% kmem_cache_free [kernel.kallsyms]
15819.00 5.2% xfs_idestroy_fork [kernel.kallsyms]
14893.00 4.9% is_bad_inode [kernel.kallsyms]
14666.00 4.9% _raw_spin_unlock_irqrestore [kernel.kallsyms]
14191.00 4.7% xfs_reclaim_inode_grab [kernel.kallsyms]
14105.00 4.7% xfs_iunlock [kernel.kallsyms]
10916.00 3.6% __ticket_spin_lock [kernel.kallsyms]
10125.00 3.4% xfs_iflush_cluster [kernel.kallsyms]
8221.00 2.7% xfs_qm_dqdetach [kernel.kallsyms]
7639.00 2.5% xfs_trans_unlocked_item [kernel.kallsyms]
7028.00 2.3% xfs_synchronize_times [kernel.kallsyms]
6974.00 2.3% up_write [kernel.kallsyms]
5870.00 1.9% call_rcu_sched [kernel.kallsyms]
5634.00 1.9% _cond_resched [kernel.kallsyms]Which is showing a similar amount of RCU overhead as the unlink as above. And this while it was doing dentry cache reclaim (~10s sample):
35921.00 15.7% __d_drop [kernel.kallsyms]
30056.00 13.1% __ticket_spin_trylock [kernel.kallsyms]
29066.00 12.7% __ticket_spin_lock [kernel.kallsyms]
19043.00 8.3% __call_rcu [kernel.kallsyms]
10098.00 4.4% iput [kernel.kallsyms]
7013.00 3.1% __shrink_dcache_sb [kernel.kallsyms]
6774.00 3.0% __percpu_counter_add [kernel.kallsyms]
6708.00 2.9% radix_tree_tag_set [kernel.kallsyms]
5362.00 2.3% xfs_inactive [kernel.kallsyms]
5130.00 2.2% __ticket_spin_unlock [kernel.kallsyms]
4884.00 2.1% call_rcu_sched [kernel.kallsyms]
4621.00 2.0% dentry_lru_del [kernel.kallsyms]
3735.00 1.6% bit_waitqueue [kernel.kallsyms]
3727.00 1.6% dentry_iput [kernel.kallsyms]
3473.00 1.5% shrink_icache_memory [kernel.kallsyms]
3279.00 1.4% kfree [kernel.kallsyms]
3101.00 1.4% xfs_perag_get [kernel.kallsyms]
2516.00 1.1% kmem_cache_free [kernel.kallsyms]
2272.00 1.0% shrink_dentry_list [kernel.kallsyms]I've never really seen any signficant dentry cache reclaim overhead in profiles of these workloads before, so this was a bit of a surprise....
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
Quoted 73 lines · Dave Chinner
Quoted 72 lines · Dave Chinner
Quoted 65 lines · Nick Piggin
Quoted 55 lines · Dave Chinner
Quoted 11 lines · Nick Piggin
git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.gitvfs-scale-workingHere is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:
* preparation patches
* introduce new locks to take overdcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking stepsStress test doing:
single thread 50M inode create
single thread rm -rf
2-way 50M inode create
2-way rm -rf
4-way 50M inode create
4-way rm -rf
8-way 50M inode create
8-way rm -rf
8-way 250M inode create
8-way rm -rfFailed about 5 minutes into the "4-way rm -rf" (~3 hours into the test) with a CPU stuck spinning on here:
[37372.084012] NMI backtrace for cpu 5 [37372.084012] CPU 5 [37372.084012] Modules linked in: [37372.084012] [37372.084012] Pid: 15214, comm: rm Not tainted 2.6.37-rc4-dgc+ #797 /Bochs [37372.084012] RIP: 0010:[<ffffffff810643c4>] [<ffffffff810643c4>] __ticket_spin_lock+0x14/0x20 [37372.084012] RSP: 0018:ffff880114643c98 EFLAGS: 00000213 [37372.084012] RAX: 0000000000008801 RBX: ffff8800687be6c0 RCX: ffff8800c4eb2688 [37372.084012] RDX: ffff880114643d38 RSI: ffff8800dfd4ea80 RDI: ffff880114643d14 [37372.084012] RBP: ffff880114643c98 R08: 0000000000000003 R09: 0000000000000000 [37372.084012] R10: 0000000000000000 R11: dead000000200200 R12: ffff880114643d14 [37372.084012] R13: ffff880114643cb8 R14: ffff880114643d38 R15: ffff8800687be71c [37372.084012] FS: 00007fd6d7c93700(0000) GS:ffff8800dfd40000(0000) knlGS:0000000000000000 [37372.084012] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [37372.084012] CR2: 0000000000bbd108 CR3: 0000000107146000 CR4: 00000000000006e0 [37372.084012] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [37372.084012] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [37372.084012] Process rm (pid: 15214, threadinfo ffff880114642000, task ffff88011b16f890) [37372.084012] Stack: [37372.084012] ffff880114643ca8 ffffffff81ad044e ffff880114643cf8 ffffffff81167ae7 [37372.084012] 0000000000000000 ffff880114643d38 000000000000000e ffff88011901d800 [37372.084012] ffff8800cdb7cf5c ffff88011901d8e0 0000000000000000 0000000000000000 [37372.084012] Call Trace: [37372.084012] [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370 [37372.084012] [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0Quoted 8 lines
[37372.084012] [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0 [37372.084012] [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff8115daa2>] dentry_unhash+0x42/0x80 [37372.084012] [<ffffffff8115db48>] vfs_rmdir+0x68/0x100 [37372.084012] [<ffffffff8115fd93>] do_rmdir+0x113/0x130 [37372.084012] [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90 [37372.084012] [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40 [37372.084012] [<ffffffff8103a002>] system_call_fastpath+0x16/0x1bQuoted 3 lines
OK good, with any luck, that's the same bug.
Is this XFS?
Yes.
Quoted 7 lines
Quoted 7 lines
Quoted 1 line
Is there any concurrent activity happening on the same dentries?
Not from an application perspective.
Quoted 1 line
Ie. are the rm -rf threads running on the same directories,
No, each thread operating on a different directory.
This is probably fixed by the same patch as the lockdep splat trace.
Quoted 96 lines
Quoted 5 lines
Quoted 1 line
or is there any reclaim happening in the background?
IIRC, kswapd was consuming about 5-10% of a CPU during parallel unlink tests. Mainly reclaiming XFS inodes, I think, but there may be dentry cache reclaim going as well.
Turns out that the kswapd peaks are upwards of 50% of a CPU for a few seconds, then idle for 10-15s. Typical perf top output of kswapd while it is active during unlinks is:
samples pcnt function DSO _______ _____ ___________________________ _________________ 17168.00 10.2% __call_rcu [kernel.kallsyms] 13223.00 7.8% kmem_cache_free [kernel.kallsyms] 12917.00 7.6% down_write [kernel.kallsyms] 12665.00 7.5% xfs_iunlock [kernel.kallsyms] 10493.00 6.2% xfs_reclaim_inode_grab [kernel.kallsyms] 9314.00 5.5% __lookup_tag [kernel.kallsyms] 9040.00 5.4% radix_tree_delete [kernel.kallsyms] 8694.00 5.1% is_bad_inode [kernel.kallsyms] 7639.00 4.5% __ticket_spin_lock [kernel.kallsyms] 6821.00 4.0% _raw_spin_unlock_irqrestore [kernel.kallsyms] 5484.00 3.2% __d_drop [kernel.kallsyms] 5114.00 3.0% xfs_reclaim_inode [kernel.kallsyms] 4626.00 2.7% __rcu_process_callbacks [kernel.kallsyms] 3556.00 2.1% up_write [kernel.kallsyms] 3206.00 1.9% _cond_resched [kernel.kallsyms] 3129.00 1.9% xfs_qm_dqdetach [kernel.kallsyms] 2327.00 1.4% radix_tree_tag_clear [kernel.kallsyms] 2327.00 1.4% call_rcu_sched [kernel.kallsyms] 2262.00 1.3% __ticket_spin_unlock [kernel.kallsyms] 2215.00 1.3% xfs_ilock [kernel.kallsyms] 2200.00 1.3% radix_tree_gang_lookup_tag [kernel.kallsyms] 1982.00 1.2% xfs_reclaim_inodes_ag [kernel.kallsyms] 1736.00 1.0% xfs_trans_unlocked_item [kernel.kallsyms] 1707.00 1.0% __ticket_spin_trylock [kernel.kallsyms] 1688.00 1.0% xfs_perag_get_tag [kernel.kallsyms] 1660.00 1.0% flat_send_IPI_mask [kernel.kallsyms] 1538.00 0.9% xfs_inode_item_destroy [kernel.kallsyms] 1312.00 0.8% __shrink_dcache_sb [kernel.kallsyms] 940.00 0.6% xfs_perag_put [kernel.kallsyms]So there is some dentry cache reclaim going on.
FWIW, it appears there is quite a lot of RCU freeing overhead (~15% more CPU time) in the work kswapd is doing during these unlinks, too. I just had a look at kswapd when a 8-way create is running - it's running at 50-60% of a cpu for seconds at a time. I caught this while it was doing pure XFS inode cache reclaim (~10s sample, kswapd reclaimed ~1M inodes):
samples pcnt function DSO _______ _____ ___________________________ _________________ 27171.00 9.0% __call_rcu [kernel.kallsyms] 21491.00 7.1% down_write [kernel.kallsyms] 20916.00 6.9% xfs_reclaim_inode [kernel.kallsyms] 20313.00 6.7% radix_tree_delete [kernel.kallsyms] 15828.00 5.3% kmem_cache_free [kernel.kallsyms] 15819.00 5.2% xfs_idestroy_fork [kernel.kallsyms] 14893.00 4.9% is_bad_inode [kernel.kallsyms] 14666.00 4.9% _raw_spin_unlock_irqrestore [kernel.kallsyms] 14191.00 4.7% xfs_reclaim_inode_grab [kernel.kallsyms] 14105.00 4.7% xfs_iunlock [kernel.kallsyms] 10916.00 3.6% __ticket_spin_lock [kernel.kallsyms] 10125.00 3.4% xfs_iflush_cluster [kernel.kallsyms] 8221.00 2.7% xfs_qm_dqdetach [kernel.kallsyms] 7639.00 2.5% xfs_trans_unlocked_item [kernel.kallsyms] 7028.00 2.3% xfs_synchronize_times [kernel.kallsyms] 6974.00 2.3% up_write [kernel.kallsyms] 5870.00 1.9% call_rcu_sched [kernel.kallsyms] 5634.00 1.9% _cond_resched [kernel.kallsyms]Which is showing a similar amount of RCU overhead as the unlink as above. And this while it was doing dentry cache reclaim (~10s sample):
35921.00 15.7% __d_drop [kernel.kallsyms] 30056.00 13.1% __ticket_spin_trylock [kernel.kallsyms] 29066.00 12.7% __ticket_spin_lock [kernel.kallsyms] 19043.00 8.3% __call_rcu [kernel.kallsyms] 10098.00 4.4% iput [kernel.kallsyms] 7013.00 3.1% __shrink_dcache_sb [kernel.kallsyms] 6774.00 3.0% __percpu_counter_add [kernel.kallsyms] 6708.00 2.9% radix_tree_tag_set [kernel.kallsyms] 5362.00 2.3% xfs_inactive [kernel.kallsyms] 5130.00 2.2% __ticket_spin_unlock [kernel.kallsyms] 4884.00 2.1% call_rcu_sched [kernel.kallsyms] 4621.00 2.0% dentry_lru_del [kernel.kallsyms] 3735.00 1.6% bit_waitqueue [kernel.kallsyms] 3727.00 1.6% dentry_iput [kernel.kallsyms] 3473.00 1.5% shrink_icache_memory [kernel.kallsyms] 3279.00 1.4% kfree [kernel.kallsyms] 3101.00 1.4% xfs_perag_get [kernel.kallsyms] 2516.00 1.1% kmem_cache_free [kernel.kallsyms] 2272.00 1.0% shrink_dentry_list [kernel.kallsyms]
Quoted 3 lines
I've never really seen any signficant dentry cache reclaim overhead in profiles of these workloads before, so this was a bit of a surprise....
call_rcu shouldn't be doing much, except for disabling irqs and linking the object into the list. I have a patch somewhere to reduce the irq disable overhead a bit, but it really shouldn't be doing a lot of work.
Sometimes you find that touching the rcu head field needs to get a cacheline exclusive, so a bit of work gets transferred there....
But it may also be something going a bit wrong in RCU. I blew it up once already, after the files_lock splitup that enabled all CPUs to create and destroy files :)
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 181 lines · Nick Piggin
Quoted 73 lines · Dave Chinner
Quoted 72 lines · Dave Chinner
Quoted 65 lines · Nick Piggin
Quoted 55 lines · Dave Chinner
Quoted 11 lines · Nick Piggin
git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.gitvfs-scale-workingHere is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:
* preparation patches
* introduce new locks to take overdcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking stepsStress test doing:
single thread 50M inode create
single thread rm -rf
2-way 50M inode create
2-way rm -rf
4-way 50M inode create
4-way rm -rf
8-way 50M inode create
8-way rm -rf
8-way 250M inode create
8-way rm -rfFailed about 5 minutes into the "4-way rm -rf" (~3 hours into the test) with a CPU stuck spinning on here:
[37372.084012] NMI backtrace for cpu 5 [37372.084012] CPU 5 [37372.084012] Modules linked in: [37372.084012] [37372.084012] Pid: 15214, comm: rm Not tainted 2.6.37-rc4-dgc+ #797 /Bochs [37372.084012] RIP: 0010:[<ffffffff810643c4>] [<ffffffff810643c4>] __ticket_spin_lock+0x14/0x20 [37372.084012] RSP: 0018:ffff880114643c98 EFLAGS: 00000213 [37372.084012] RAX: 0000000000008801 RBX: ffff8800687be6c0 RCX: ffff8800c4eb2688 [37372.084012] RDX: ffff880114643d38 RSI: ffff8800dfd4ea80 RDI: ffff880114643d14 [37372.084012] RBP: ffff880114643c98 R08: 0000000000000003 R09: 0000000000000000 [37372.084012] R10: 0000000000000000 R11: dead000000200200 R12: ffff880114643d14 [37372.084012] R13: ffff880114643cb8 R14: ffff880114643d38 R15: ffff8800687be71c [37372.084012] FS: 00007fd6d7c93700(0000) GS:ffff8800dfd40000(0000) knlGS:0000000000000000 [37372.084012] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [37372.084012] CR2: 0000000000bbd108 CR3: 0000000107146000 CR4: 00000000000006e0 [37372.084012] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [37372.084012] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [37372.084012] Process rm (pid: 15214, threadinfo ffff880114642000, task ffff88011b16f890) [37372.084012] Stack: [37372.084012] ffff880114643ca8 ffffffff81ad044e ffff880114643cf8 ffffffff81167ae7 [37372.084012] 0000000000000000 ffff880114643d38 000000000000000e ffff88011901d800 [37372.084012] ffff8800cdb7cf5c ffff88011901d8e0 0000000000000000 0000000000000000 [37372.084012] Call Trace: [37372.084012] [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370 [37372.084012] [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0Quoted 8 lines
[37372.084012] [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0 [37372.084012] [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff8115daa2>] dentry_unhash+0x42/0x80 [37372.084012] [<ffffffff8115db48>] vfs_rmdir+0x68/0x100 [37372.084012] [<ffffffff8115fd93>] do_rmdir+0x113/0x130 [37372.084012] [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90 [37372.084012] [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40 [37372.084012] [<ffffffff8103a002>] system_call_fastpath+0x16/0x1bQuoted 3 lines
OK good, with any luck, that's the same bug.
Is this XFS?
Yes.
Quoted 7 lines
Quoted 7 lines
Quoted 1 line
Is there any concurrent activity happening on the same dentries?
Not from an application perspective.
Quoted 1 line
Ie. are the rm -rf threads running on the same directories,
No, each thread operating on a different directory.
This is probably fixed by the same patch as the lockdep splat trace.
Quoted 96 lines
Quoted 5 lines
Quoted 1 line
or is there any reclaim happening in the background?
IIRC, kswapd was consuming about 5-10% of a CPU during parallel unlink tests. Mainly reclaiming XFS inodes, I think, but there may be dentry cache reclaim going as well.
Turns out that the kswapd peaks are upwards of 50% of a CPU for a few seconds, then idle for 10-15s. Typical perf top output of kswapd while it is active during unlinks is:
samples pcnt function DSO _______ _____ ___________________________ _________________ 17168.00 10.2% __call_rcu [kernel.kallsyms] 13223.00 7.8% kmem_cache_free [kernel.kallsyms] 12917.00 7.6% down_write [kernel.kallsyms] 12665.00 7.5% xfs_iunlock [kernel.kallsyms] 10493.00 6.2% xfs_reclaim_inode_grab [kernel.kallsyms] 9314.00 5.5% __lookup_tag [kernel.kallsyms] 9040.00 5.4% radix_tree_delete [kernel.kallsyms] 8694.00 5.1% is_bad_inode [kernel.kallsyms] 7639.00 4.5% __ticket_spin_lock [kernel.kallsyms] 6821.00 4.0% _raw_spin_unlock_irqrestore [kernel.kallsyms] 5484.00 3.2% __d_drop [kernel.kallsyms] 5114.00 3.0% xfs_reclaim_inode [kernel.kallsyms] 4626.00 2.7% __rcu_process_callbacks [kernel.kallsyms] 3556.00 2.1% up_write [kernel.kallsyms] 3206.00 1.9% _cond_resched [kernel.kallsyms] 3129.00 1.9% xfs_qm_dqdetach [kernel.kallsyms] 2327.00 1.4% radix_tree_tag_clear [kernel.kallsyms] 2327.00 1.4% call_rcu_sched [kernel.kallsyms] 2262.00 1.3% __ticket_spin_unlock [kernel.kallsyms] 2215.00 1.3% xfs_ilock [kernel.kallsyms] 2200.00 1.3% radix_tree_gang_lookup_tag [kernel.kallsyms] 1982.00 1.2% xfs_reclaim_inodes_ag [kernel.kallsyms] 1736.00 1.0% xfs_trans_unlocked_item [kernel.kallsyms] 1707.00 1.0% __ticket_spin_trylock [kernel.kallsyms] 1688.00 1.0% xfs_perag_get_tag [kernel.kallsyms] 1660.00 1.0% flat_send_IPI_mask [kernel.kallsyms] 1538.00 0.9% xfs_inode_item_destroy [kernel.kallsyms] 1312.00 0.8% __shrink_dcache_sb [kernel.kallsyms] 940.00 0.6% xfs_perag_put [kernel.kallsyms]So there is some dentry cache reclaim going on.
FWIW, it appears there is quite a lot of RCU freeing overhead (~15% more CPU time) in the work kswapd is doing during these unlinks, too. I just had a look at kswapd when a 8-way create is running - it's running at 50-60% of a cpu for seconds at a time. I caught this while it was doing pure XFS inode cache reclaim (~10s sample, kswapd reclaimed ~1M inodes):
samples pcnt function DSO _______ _____ ___________________________ _________________ 27171.00 9.0% __call_rcu [kernel.kallsyms] 21491.00 7.1% down_write [kernel.kallsyms] 20916.00 6.9% xfs_reclaim_inode [kernel.kallsyms] 20313.00 6.7% radix_tree_delete [kernel.kallsyms] 15828.00 5.3% kmem_cache_free [kernel.kallsyms] 15819.00 5.2% xfs_idestroy_fork [kernel.kallsyms] 14893.00 4.9% is_bad_inode [kernel.kallsyms] 14666.00 4.9% _raw_spin_unlock_irqrestore [kernel.kallsyms] 14191.00 4.7% xfs_reclaim_inode_grab [kernel.kallsyms] 14105.00 4.7% xfs_iunlock [kernel.kallsyms] 10916.00 3.6% __ticket_spin_lock [kernel.kallsyms] 10125.00 3.4% xfs_iflush_cluster [kernel.kallsyms] 8221.00 2.7% xfs_qm_dqdetach [kernel.kallsyms] 7639.00 2.5% xfs_trans_unlocked_item [kernel.kallsyms] 7028.00 2.3% xfs_synchronize_times [kernel.kallsyms] 6974.00 2.3% up_write [kernel.kallsyms] 5870.00 1.9% call_rcu_sched [kernel.kallsyms] 5634.00 1.9% _cond_resched [kernel.kallsyms]Which is showing a similar amount of RCU overhead as the unlink as above. And this while it was doing dentry cache reclaim (~10s sample):
35921.00 15.7% __d_drop [kernel.kallsyms] 30056.00 13.1% __ticket_spin_trylock [kernel.kallsyms] 29066.00 12.7% __ticket_spin_lock [kernel.kallsyms] 19043.00 8.3% __call_rcu [kernel.kallsyms] 10098.00 4.4% iput [kernel.kallsyms] 7013.00 3.1% __shrink_dcache_sb [kernel.kallsyms] 6774.00 3.0% __percpu_counter_add [kernel.kallsyms] 6708.00 2.9% radix_tree_tag_set [kernel.kallsyms] 5362.00 2.3% xfs_inactive [kernel.kallsyms] 5130.00 2.2% __ticket_spin_unlock [kernel.kallsyms] 4884.00 2.1% call_rcu_sched [kernel.kallsyms] 4621.00 2.0% dentry_lru_del [kernel.kallsyms] 3735.00 1.6% bit_waitqueue [kernel.kallsyms] 3727.00 1.6% dentry_iput [kernel.kallsyms] 3473.00 1.5% shrink_icache_memory [kernel.kallsyms] 3279.00 1.4% kfree [kernel.kallsyms] 3101.00 1.4% xfs_perag_get [kernel.kallsyms] 2516.00 1.1% kmem_cache_free [kernel.kallsyms] 2272.00 1.0% shrink_dentry_list [kernel.kallsyms]
Quoted 7 lines
Quoted 3 lines
I've never really seen any signficant dentry cache reclaim overhead in profiles of these workloads before, so this was a bit of a surprise....
call_rcushouldn't be doing much, except for disabling irqs and linking the object into the list. I have a patch somewhere to reduce the irq disable overhead a bit, but it really shouldn't be doing a lot of work.
Could you please enable CONFIG_RCU_TRACE, mount debugfs somewhere, and look at rcu/rcudata? There will be a "ql=" number printed for each CPU, and if that number is too large, __call_rcu() does take what it considers to be corrective action, which can incur some overhead.
If this is the problem, then increasing the value of the qhimark module parameter might help.
If this is not the problem, I could make a patch that disables some of __call_rcu()'s grace-period acceleration code if you are willing to try it out.
Quoted 6 lines
Sometimes you find that touching the rcu head field needs to get a cacheline exclusive, so a bit of work gets transferred there....
But it may also be something going a bit wrong in RCU. I blew it up once already, after the
files_locksplitup that enabled all CPUs to create and destroy files :)
I would certainly like the opportunity to fix any bugs that might be in RCU... ;-)
Thanx, Paul
Quoted 195 lines · Paul E. McKenney
Quoted 181 lines · Nick Piggin
Quoted 73 lines · Dave Chinner
Quoted 72 lines · Dave Chinner
Quoted 65 lines · Nick Piggin
Quoted 55 lines · Dave Chinner
Quoted 11 lines · Nick Piggin
git://git.kernel.org/pub/scm/linux/kernel/git/npiggin/linux-npiggin.gitvfs-scale-workingHere is an new set of vfs patches for review, not that there was much interest last time they were posted. It is structured like:
* preparation patches
* introduce new locks to take overdcache_lock, then remove it
* cleaning up and reworking things for new locks
* rcu-walk path walking
* start on some fine grained locking stepsStress test doing:
single thread 50M inode create
single thread rm -rf
2-way 50M inode create
2-way rm -rf
4-way 50M inode create
4-way rm -rf
8-way 50M inode create
8-way rm -rf
8-way 250M inode create
8-way rm -rfFailed about 5 minutes into the "4-way rm -rf" (~3 hours into the test) with a CPU stuck spinning on here:
[37372.084012] NMI backtrace for cpu 5 [37372.084012] CPU 5 [37372.084012] Modules linked in: [37372.084012] [37372.084012] Pid: 15214, comm: rm Not tainted 2.6.37-rc4-dgc+ #797 /Bochs [37372.084012] RIP: 0010:[<ffffffff810643c4>] [<ffffffff810643c4>] __ticket_spin_lock+0x14/0x20 [37372.084012] RSP: 0018:ffff880114643c98 EFLAGS: 00000213 [37372.084012] RAX: 0000000000008801 RBX: ffff8800687be6c0 RCX: ffff8800c4eb2688 [37372.084012] RDX: ffff880114643d38 RSI: ffff8800dfd4ea80 RDI: ffff880114643d14 [37372.084012] RBP: ffff880114643c98 R08: 0000000000000003 R09: 0000000000000000 [37372.084012] R10: 0000000000000000 R11: dead000000200200 R12: ffff880114643d14 [37372.084012] R13: ffff880114643cb8 R14: ffff880114643d38 R15: ffff8800687be71c [37372.084012] FS: 00007fd6d7c93700(0000) GS:ffff8800dfd40000(0000) knlGS:0000000000000000 [37372.084012] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [37372.084012] CR2: 0000000000bbd108 CR3: 0000000107146000 CR4: 00000000000006e0 [37372.084012] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [37372.084012] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [37372.084012] Process rm (pid: 15214, threadinfo ffff880114642000, task ffff88011b16f890) [37372.084012] Stack: [37372.084012] ffff880114643ca8 ffffffff81ad044e ffff880114643cf8 ffffffff81167ae7 [37372.084012] 0000000000000000 ffff880114643d38 000000000000000e ffff88011901d800 [37372.084012] ffff8800cdb7cf5c ffff88011901d8e0 0000000000000000 0000000000000000 [37372.084012] Call Trace: [37372.084012] [<ffffffff81ad044e>] _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff81167ae7>] shrink_dentry_list+0x47/0x370 [37372.084012] [<ffffffff81167f5e>] __shrink_dcache_sb+0x14e/0x1e0Quoted 8 lines
[37372.084012] [<ffffffff81168456>] shrink_dcache_parent+0x276/0x2d0 [37372.084012] [<ffffffff81ad044e>] ? _raw_spin_lock+0xe/0x20 [37372.084012] [<ffffffff8115daa2>] dentry_unhash+0x42/0x80 [37372.084012] [<ffffffff8115db48>] vfs_rmdir+0x68/0x100 [37372.084012] [<ffffffff8115fd93>] do_rmdir+0x113/0x130 [37372.084012] [<ffffffff8114f5ad>] ? filp_close+0x5d/0x90 [37372.084012] [<ffffffff8115fde5>] sys_unlinkat+0x35/0x40 [37372.084012] [<ffffffff8103a002>] system_call_fastpath+0x16/0x1bQuoted 3 lines
OK good, with any luck, that's the same bug.
Is this XFS?
Yes.
Quoted 7 lines
Quoted 7 lines
Quoted 1 line
Is there any concurrent activity happening on the same dentries?
Not from an application perspective.
Quoted 1 line
Ie. are the rm -rf threads running on the same directories,
No, each thread operating on a different directory.
This is probably fixed by the same patch as the lockdep splat trace.
Quoted 96 lines
Quoted 5 lines
Quoted 1 line
or is there any reclaim happening in the background?
IIRC, kswapd was consuming about 5-10% of a CPU during parallel unlink tests. Mainly reclaiming XFS inodes, I think, but there may be dentry cache reclaim going as well.
Turns out that the kswapd peaks are upwards of 50% of a CPU for a few seconds, then idle for 10-15s. Typical perf top output of kswapd while it is active during unlinks is:
samples pcnt function DSO _______ _____ ___________________________ _________________ 17168.00 10.2% __call_rcu [kernel.kallsyms] 13223.00 7.8% kmem_cache_free [kernel.kallsyms] 12917.00 7.6% down_write [kernel.kallsyms] 12665.00 7.5% xfs_iunlock [kernel.kallsyms] 10493.00 6.2% xfs_reclaim_inode_grab [kernel.kallsyms] 9314.00 5.5% __lookup_tag [kernel.kallsyms] 9040.00 5.4% radix_tree_delete [kernel.kallsyms] 8694.00 5.1% is_bad_inode [kernel.kallsyms] 7639.00 4.5% __ticket_spin_lock [kernel.kallsyms] 6821.00 4.0% _raw_spin_unlock_irqrestore [kernel.kallsyms] 5484.00 3.2% __d_drop [kernel.kallsyms] 5114.00 3.0% xfs_reclaim_inode [kernel.kallsyms] 4626.00 2.7% __rcu_process_callbacks [kernel.kallsyms] 3556.00 2.1% up_write [kernel.kallsyms] 3206.00 1.9% _cond_resched [kernel.kallsyms] 3129.00 1.9% xfs_qm_dqdetach [kernel.kallsyms] 2327.00 1.4% radix_tree_tag_clear [kernel.kallsyms] 2327.00 1.4% call_rcu_sched [kernel.kallsyms] 2262.00 1.3% __ticket_spin_unlock [kernel.kallsyms] 2215.00 1.3% xfs_ilock [kernel.kallsyms] 2200.00 1.3% radix_tree_gang_lookup_tag [kernel.kallsyms] 1982.00 1.2% xfs_reclaim_inodes_ag [kernel.kallsyms] 1736.00 1.0% xfs_trans_unlocked_item [kernel.kallsyms] 1707.00 1.0% __ticket_spin_trylock [kernel.kallsyms] 1688.00 1.0% xfs_perag_get_tag [kernel.kallsyms] 1660.00 1.0% flat_send_IPI_mask [kernel.kallsyms] 1538.00 0.9% xfs_inode_item_destroy [kernel.kallsyms] 1312.00 0.8% __shrink_dcache_sb [kernel.kallsyms] 940.00 0.6% xfs_perag_put [kernel.kallsyms]So there is some dentry cache reclaim going on.
FWIW, it appears there is quite a lot of RCU freeing overhead (~15% more CPU time) in the work kswapd is doing during these unlinks, too. I just had a look at kswapd when a 8-way create is running - it's running at 50-60% of a cpu for seconds at a time. I caught this while it was doing pure XFS inode cache reclaim (~10s sample, kswapd reclaimed ~1M inodes):
samples pcnt function DSO _______ _____ ___________________________ _________________ 27171.00 9.0% __call_rcu [kernel.kallsyms] 21491.00 7.1% down_write [kernel.kallsyms] 20916.00 6.9% xfs_reclaim_inode [kernel.kallsyms] 20313.00 6.7% radix_tree_delete [kernel.kallsyms] 15828.00 5.3% kmem_cache_free [kernel.kallsyms] 15819.00 5.2% xfs_idestroy_fork [kernel.kallsyms] 14893.00 4.9% is_bad_inode [kernel.kallsyms] 14666.00 4.9% _raw_spin_unlock_irqrestore [kernel.kallsyms] 14191.00 4.7% xfs_reclaim_inode_grab [kernel.kallsyms] 14105.00 4.7% xfs_iunlock [kernel.kallsyms] 10916.00 3.6% __ticket_spin_lock [kernel.kallsyms] 10125.00 3.4% xfs_iflush_cluster [kernel.kallsyms] 8221.00 2.7% xfs_qm_dqdetach [kernel.kallsyms] 7639.00 2.5% xfs_trans_unlocked_item [kernel.kallsyms] 7028.00 2.3% xfs_synchronize_times [kernel.kallsyms] 6974.00 2.3% up_write [kernel.kallsyms] 5870.00 1.9% call_rcu_sched [kernel.kallsyms] 5634.00 1.9% _cond_resched [kernel.kallsyms]Which is showing a similar amount of RCU overhead as the unlink as above. And this while it was doing dentry cache reclaim (~10s sample):
35921.00 15.7% __d_drop [kernel.kallsyms] 30056.00 13.1% __ticket_spin_trylock [kernel.kallsyms] 29066.00 12.7% __ticket_spin_lock [kernel.kallsyms] 19043.00 8.3% __call_rcu [kernel.kallsyms] 10098.00 4.4% iput [kernel.kallsyms] 7013.00 3.1% __shrink_dcache_sb [kernel.kallsyms] 6774.00 3.0% __percpu_counter_add [kernel.kallsyms] 6708.00 2.9% radix_tree_tag_set [kernel.kallsyms] 5362.00 2.3% xfs_inactive [kernel.kallsyms] 5130.00 2.2% __ticket_spin_unlock [kernel.kallsyms] 4884.00 2.1% call_rcu_sched [kernel.kallsyms] 4621.00 2.0% dentry_lru_del [kernel.kallsyms] 3735.00 1.6% bit_waitqueue [kernel.kallsyms] 3727.00 1.6% dentry_iput [kernel.kallsyms] 3473.00 1.5% shrink_icache_memory [kernel.kallsyms] 3279.00 1.4% kfree [kernel.kallsyms] 3101.00 1.4% xfs_perag_get [kernel.kallsyms] 2516.00 1.1% kmem_cache_free [kernel.kallsyms] 2272.00 1.0% shrink_dentry_list [kernel.kallsyms]Quoted 7 lines
Quoted 3 lines
I've never really seen any signficant dentry cache reclaim overhead in profiles of these workloads before, so this was a bit of a surprise....
call_rcushouldn't be doing much, except for disabling irqs and linking the object into the list. I have a patch somewhere to reduce the irq disable overhead a bit, but it really shouldn't be doing a lot of work.Could you please enable
CONFIG_RCU_TRACE, mount debugfs somewhere, and look at rcu/rcudata? There will be a "ql=" number printed for each CPU, and if that number is too large,__call_rcu()does take what it considers to be corrective action, which can incur some overhead.
Quoted 6 lines
If this is the problem, then increasing the value of the qhimark module parameter might help.
If this is not the problem, I could make a patch that disables some of __call_rcu()'s grace-period acceleration code if you are willing to try it out.
Another thing that might help is to reduce the value of CONFIG_RCU_FANOUT to something like 16. If this does help, then there is a reasonably straightforward change I can make to RCU.
Thanx, Paul
Quoted 11 lines
Quoted 6 lines
Sometimes you find that touching the rcu head field needs to get a cacheline exclusive, so a bit of work gets transferred there....
But it may also be something going a bit wrong in RCU. I blew it up once already, after the
files_locksplitup that enabled all CPUs to create and destroy files :)I would certainly like the opportunity to fix any bugs that might be in RCU... ;-)
Thanx, Paul
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html