Skip to contentSkip to conversation

FastWalk Dcache back port from 2.5

PATCH In review Hanna Linder 2 participants 3 messages May 9, 2002 – May 10, 2002

Subsystems (2)
VFS exec and binfmt
lore
Share & tools
lore

Hanna Linder

author
#0 lore raw

PATCH 1/1 FastWalk Dcache back port from 2.5

Hello,

Please consider this new and improved fast walk patch for
inclusion in your 2.4.19 tree. This patch reduces cacheline bouncing due to numerous atomic increments and decrements of the d_count reference counter during path walking by holding the dcache_lock as long as the dentries are in the cache.
Linus included my original patch in 2.5.11. Al Viro made a
few great changes in 2.5.12 and Paul Menage added one fix. The following patch includes their changes as well. This code has been in 2.5 for almost 3 weeks and appears to be quite stable. In addition it shows a throughput improvement while running dbench on an 8-way. Here is the output:

http://west.dl.sourceforge.net/sourceforge/lse/2419fw.png

Hanna Linder
[email protected]
IBM Linux Technology Center

Patch is also available at:
http://west.dl.sourceforge.net/sourceforge/lse/fast_walkA3-2.4.19-pre8.patch

---------

fs/dcache.c +13 -4
diff -Nru -X dontdiff linux-2.4.19-pre8/fs/dcache.c linux-fastwalk/fs/dcache.c
@@ -702,13 +702,25 @@
  
 struct dentry * d_lookup(struct dentry * parent, struct qstr * name)
 {
+	struct dentry *dentry = NULL;
+
+	spin_lock(&dcache_lock);
+	dentry = __d_lookup(parent,name);
+	if (dentry)
+		__dget_locked(dentry);
+	spin_unlock(&dcache_lock);
+	return dentry;
+}
+
+struct dentry * __d_lookup(struct dentry * parent, struct qstr * name)  
+{
+
 	unsigned int len = name->len;
 	unsigned int hash = name->hash;
 	const unsigned char *str = name->name;
 	struct list_head *head = d_hash(parent,hash);
 	struct list_head *tmp;
 
-	spin_lock(&dcache_lock);
 	tmp = head->next;
 	for (;;) {
 		struct dentry * dentry = list_entry(tmp, struct dentry, d_hash);

@@ -728,12 +740,9 @@
 			if (memcmp(dentry->d_name.name, str, len))
 				continue;
 		}
-		__dget_locked(dentry);
 		dentry->d_vfs_flags |= DCACHE_REFERENCED;
-		spin_unlock(&dcache_lock);
 		return dentry;
 	}
-	spin_unlock(&dcache_lock);
 	return NULL;
 }
fs/exec.c +1 -2
diff -Nru -X dontdiff linux-2.4.19-pre8/fs/exec.c linux-fastwalk/fs/exec.c
@@ -343,8 +343,7 @@
 	struct file *file;
 	int err = 0;
 
-	if (path_init(name, LOOKUP_FOLLOW|LOOKUP_POSITIVE, &nd))
-		err = path_walk(name, &nd);
+	err = path_lookup(name, LOOKUP_FOLLOW|LOOKUP_POSITIVE, &nd);
 	file = ERR_PTR(err);
 	if (!err) {
 		inode = nd.dentry->d_inode;
fs/namei.c +236 -120

579 lines not on this page · raw message

fs/namespace.c +10 -21
diff -Nru -X dontdiff linux-2.4.19-pre8/fs/namespace.c linux-fastwalk/fs/namespace.c
@@ -23,6 +23,7 @@
 
 struct vfsmount *do_kern_mount(const char *type, int flags, char *name, void *data);
 int do_remount_sb(struct super_block *sb, int flags, void * data);
+int __init init_rootfs(void);
 void kill_super(struct super_block *sb);
 
 static struct list_head *mount_hashtable;

@@ -367,8 +368,7 @@
 	if (IS_ERR(kname))
 		goto out;
 	retval = 0;
-	if (path_init(kname, LOOKUP_POSITIVE|LOOKUP_FOLLOW, &nd))
-		retval = path_walk(kname, &nd);
+	retval = path_lookup(kname, LOOKUP_POSITIVE|LOOKUP_FOLLOW, &nd); 
 	putname(kname);
 	if (retval)
 		goto out;

@@ -496,8 +496,7 @@
 		return err;
 	if (!old_name || !*old_name)
 		return -EINVAL;
-	if (path_init(old_name, LOOKUP_POSITIVE|LOOKUP_FOLLOW, &old_nd))
-		err = path_walk(old_name, &old_nd);
+	err = path_lookup(old_name, LOOKUP_POSITIVE|LOOKUP_FOLLOW, &old_nd);
 	if (err)
 		return err;
 

@@ -563,8 +562,7 @@
 		return -EPERM;
 	if (!old_name || !*old_name)
 		return -EINVAL;
-	if (path_init(old_name, LOOKUP_POSITIVE|LOOKUP_FOLLOW, &old_nd))
-		err = path_walk(old_name, &old_nd);
+	err = path_lookup(old_name, LOOKUP_POSITIVE|LOOKUP_FOLLOW, &old_nd);
 	if (err)
 		return err;
 

@@ -730,8 +728,7 @@
 	flags &= ~(MS_NOSUID|MS_NOEXEC|MS_NODEV);
 
 	/* ... and get the mountpoint */
-	if (path_init(dir_name, LOOKUP_FOLLOW|LOOKUP_POSITIVE, &nd))
-		retval = path_walk(dir_name, &nd);
+	retval = path_lookup(dir_name, LOOKUP_FOLLOW|LOOKUP_POSITIVE, &nd);
 	if (retval)
 		return retval;
 

@@ -923,8 +920,7 @@
 	if (IS_ERR(name))
 		goto out0;
 	error = 0;
-	if (path_init(name, LOOKUP_POSITIVE|LOOKUP_FOLLOW|LOOKUP_DIRECTORY, &new_nd))
-		error = path_walk(name, &new_nd);
+	error = path_lookup(name, LOOKUP_POSITIVE|LOOKUP_FOLLOW|LOOKUP_DIRECTORY, &new_nd);
 	putname(name);
 	if (error)
 		goto out0;

@@ -937,8 +933,7 @@
 	if (IS_ERR(name))
 		goto out1;
 	error = 0;
-	if (path_init(name, LOOKUP_POSITIVE|LOOKUP_FOLLOW|LOOKUP_DIRECTORY, &old_nd))
-		error = path_walk(name, &old_nd);
+	error = path_lookup(name, LOOKUP_POSITIVE|LOOKUP_FOLLOW|LOOKUP_DIRECTORY, &old_nd);
 	putname(name);
 	if (error)
 		goto out1;

@@ -1047,15 +1042,9 @@
 	if (!mnt_cache)
 		panic("Cannot create vfsmount cache");
 
-	mempages >>= (16 - PAGE_SHIFT);
-	mempages *= sizeof(struct list_head);
-	for (order = 0; ((1UL << order) << PAGE_SHIFT) < mempages; order++)
-		;
-
-	do {
-		mount_hashtable = (struct list_head *)
-			__get_free_pages(GFP_ATOMIC, order);
-	} while (mount_hashtable == NULL && --order >= 0);
+	order = 0;
+	mount_hashtable = (struct list_head *)
+		__get_free_pages(GFP_ATOMIC, order);
 
 	if (!mount_hashtable)
 		panic("Failed to allocate mount hash table\n");
fs/open.c +2 -4
diff -Nru -X dontdiff linux-2.4.19-pre8/fs/open.c linux-fastwalk/fs/open.c
@@ -392,8 +392,7 @@
 		goto out;
 
 	error = 0;
-	if (path_init(name,LOOKUP_POSITIVE|LOOKUP_FOLLOW|LOOKUP_DIRECTORY,&nd))
-		error = path_walk(name, &nd);
+	error = path_lookup(name,LOOKUP_POSITIVE|LOOKUP_FOLLOW|LOOKUP_DIRECTORY,&nd);
 	putname(name);
 	if (error)
 		goto out;

@@ -451,9 +450,8 @@
 	if (IS_ERR(name))
 		goto out;
 
-	path_init(name, LOOKUP_POSITIVE | LOOKUP_FOLLOW |
+	error = path_lookup(name, LOOKUP_POSITIVE | LOOKUP_FOLLOW |
 		      LOOKUP_DIRECTORY | LOOKUP_NOALT, &nd);
-	error = path_walk(name, &nd);	
 	putname(name);
 	if (error)
 		goto out;
fs/super.c +1 -2
diff -Nru -X dontdiff linux-2.4.19-pre8/fs/super.c linux-fastwalk/fs/super.c
@@ -659,8 +659,7 @@
 	/* What device it is? */
 	if (!dev_name || !*dev_name)
 		return ERR_PTR(-EINVAL);
-	if (path_init(dev_name, LOOKUP_FOLLOW|LOOKUP_POSITIVE, &nd))
-		error = path_walk(dev_name, &nd);
+	error = path_lookup(dev_name, LOOKUP_FOLLOW|LOOKUP_POSITIVE, &nd);
 	if (error)
 		return ERR_PTR(error);
 	inode = nd.dentry->d_inode;
include/linux/dcache.h +1 -0
diff -Nru -X dontdiff linux-2.4.19-pre8/include/linux/dcache.h linux-fastwalk/include/linux/dcache.h
@@ -218,6 +218,7 @@
 
 /* appendix may either be NULL or be used for transname suffixes */
 extern struct dentry * d_lookup(struct dentry *, struct qstr *);
+extern struct dentry * __d_lookup(struct dentry *, struct qstr *);
 
 /* validate "insecure" dentry pointer */
 extern int d_validate(struct dentry *, struct dentry *);
include/linux/fs.h +5 -0
diff -Nru -X dontdiff linux-2.4.19-pre8/include/linux/fs.h linux-fastwalk/include/linux/fs.h
@@ -653,6 +653,8 @@
 	struct qstr last;
 	unsigned int flags;
 	int last_type;
+	struct dentry *old_dentry;
+	struct vfsmount *old_mnt;
 };
 
 #define DQUOT_USR_ENABLED	0x01		/* User diskquotas enabled */

@@ -1289,6 +1291,7 @@
  *  - require a directory
  *  - ending slashes ok even for nonexistent files
  *  - internal "there are more path compnents" flag
+ *  - locked when lookup done with dcache_lock held
  */
 #define LOOKUP_FOLLOW		(1)
 #define LOOKUP_DIRECTORY	(2)

@@ -1296,6 +1299,7 @@
 #define LOOKUP_POSITIVE		(8)
 #define LOOKUP_PARENT		(16)
 #define LOOKUP_NOALT		(32)
+
 /*
  * Type of the last component on LOOKUP_PARENT
  */

@@ -1325,6 +1329,7 @@
 extern int FASTCALL(__user_walk(const char *, unsigned, struct nameidata *));
 extern int FASTCALL(path_init(const char *, unsigned, struct nameidata *));
 extern int FASTCALL(path_walk(const char *, struct nameidata *));
+extern int FASTCALL(path_lookup(const char *, unsigned, struct nameidata *));
 extern int FASTCALL(link_path_walk(const char *, struct nameidata *));
 extern void path_release(struct nameidata *);
 extern int follow_down(struct vfsmount **, struct dentry **);
include/linux/fs_struct.h +4 -0
diff -Nru -X dontdiff linux-2.4.19-pre8/include/linux/fs_struct.h linux-fastwalk/include/linux/fs_struct.h
@@ -32,10 +32,12 @@
 	struct dentry *old_root;
 	struct vfsmount *old_rootmnt;
 	write_lock(&fs->lock);
+	spin_lock(&dcache_lock);
 	old_root = fs->root;
 	old_rootmnt = fs->rootmnt;
 	fs->rootmnt = mntget(mnt);
 	fs->root = dget(dentry);
+	spin_unlock(&dcache_lock);
 	write_unlock(&fs->lock);
 	if (old_root) {
 		dput(old_root);

@@ -55,10 +57,12 @@
 	struct dentry *old_pwd;
 	struct vfsmount *old_pwdmnt;
 	write_lock(&fs->lock);
+	spin_lock(&dcache_lock);
 	old_pwd = fs->pwd;
 	old_pwdmnt = fs->pwdmnt;
 	fs->pwdmnt = mntget(mnt);
 	fs->pwd = dget(dentry);
+	spin_unlock(&dcache_lock);
 	write_unlock(&fs->lock);
 	if (old_pwd) {
 		dput(old_pwd);
kernel/ksyms.c +1 -0
diff -Nru -X dontdiff linux-2.4.19-pre8/kernel/ksyms.c linux-fastwalk/kernel/ksyms.c
@@ -144,6 +144,7 @@
 EXPORT_SYMBOL(lookup_mnt);
 EXPORT_SYMBOL(path_init);
 EXPORT_SYMBOL(path_walk);
+EXPORT_SYMBOL(path_lookup);
 EXPORT_SYMBOL(path_release);
 EXPORT_SYMBOL(__user_walk);
 EXPORT_SYMBOL(lookup_one_len);
#1 lore raw
Quoted 11 lines · Hanna Linder

Hello,

Please consider this new and improved fast walk patch for
inclusion in your 2.4.19 tree. This patch reduces cacheline bouncing due to numerous atomic increments and decrements of the d_count reference counter during path walking by holding the dcache_lock as long as the dentries are in the cache.
Linus included my original patch in 2.5.11. Al Viro made a
few great changes in 2.5.12 and Paul Menage added one fix. The following patch includes their changes as well. This code has been in 2.5 for almost 3 weeks and appears to be quite stable.

The fixed code isn't there for a long time yet.. I'd rather see the path_lookup cleanup which already is in -ac go into 2.4.19 (and if I understood Marcelo right that'll be release in the next time) and wait for 2.4.20-pre<early> for the massive changes.

Just my .5 (Euro-)Cent.

author
#2 lore raw
Quoted 2 lines · --On Friday, May 10, 2002 12:58 AM +0100 Christoph Hellwig <[email protected]>…

path_lookup cleanup which already is in -ac go into 2.4.19 (and if I understood Marcelo right that'll be release in the next time) and wait

Marcelo,

I can submit a new path_lookup against the most recent kernel if you would like.

Thanks.

Hanna