Skip to contentSkip to conversation

reduce NFS stack usage

PATCH Steve Dickson 3 participants 6 messages Sep 25, 2003 – Sep 30, 2003 2 messages not in the archive

Subsystems (2)
lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

#1 lore raw

Hey Jeff,

Question:
Why are only nfs_lookup_revalidate() and nfs_readdir() a problem and not the other 4 ops (like nfs_lookup())? Is the case only those two showed up in the stack overflow oops trace?

Also, not like there much choice in matter, but I wonder what type of performance hit (if any) there will be by making these routines call kmalloc()... lookups and readdirs are pretty popular ops...

SteveD.

Quoted 147 lines · Jeff Garzik

David Mansfield spotted a bug in a memset() call, in my initial patch.

Attached is an updated version. The only change is s/sizeof(my_entry)/sizeof(*my_entry)/ in the memset.

Jeff

------------------------------------------------------------------------

===== fs/nfs/dir.c 1.12 vs edited =====

fs/nfs/dir.c +51 -20
@@ -349,14 +349,25 @@
 {
 	struct dentry	*dentry = filp->f_dentry;
 	struct inode	*inode = dentry->d_inode;
-	nfs_readdir_descriptor_t my_desc,
-			*desc = &my_desc;
-	struct nfs_entry my_entry;
+	nfs_readdir_descriptor_t *desc;
+	struct nfs_entry *my_entry;
 	long		res;
+	int		rc = -EINVAL;
+	void		*mem;
+
+	mem = kmalloc(sizeof(struct nfs_entry) +
+		      sizeof(nfs_readdir_descriptor_t), GFP_KERNEL);
+	if (!mem)
+		return -ENOMEM;
+
+	my_entry = mem;
+	desc = mem + sizeof(struct nfs_entry);
 
 	res = nfs_revalidate(dentry);
-	if (res < 0)
-		return res;
+	if (res < 0) {
+		rc = (int) res;
+		goto out;
+	}
 
 	/*
 	 * filp->f_pos points to the file offset in the page cache.
@@ -365,11 +376,11 @@
 	 * itself.
 	 */
 	memset(desc, 0, sizeof(*desc));
-	memset(&my_entry, 0, sizeof(my_entry));
+	memset(my_entry, 0, sizeof(*my_entry));
 
 	desc->file = filp;
 	desc->target = filp->f_pos;
-	desc->entry = &my_entry;
+	desc->entry = my_entry;
 	desc->decode = NFS_PROTO(inode)->decode_dirent;
 
 	while(!desc->entry->eof) {
@@ -393,11 +404,16 @@
 			break;
 		}
 	}
+
 	if (desc->error < 0)
-		return desc->error;
-	if (res < 0)
-		return res;
-	return 0;
+		rc = desc->error;
+	else if (res < 0)
+		rc = res;
+	/* fall through */
+
+out:
+	kfree(mem);
+	return rc;
 }
 
 /*
@@ -476,13 +492,22 @@
 	struct inode *dir;
 	struct inode *inode;
 	int error;
-	struct nfs_fh fhandle;
-	struct nfs_fattr fattr;
+	struct nfs_fh *fhandle;
+	struct nfs_fattr *fattr;
+	void *mem;
+	int rc = 0;
 
 	lock_kernel();
 	dir = dentry->d_parent->d_inode;
 	inode = dentry->d_inode;
 
+	mem = kmalloc(sizeof(struct nfs_fh) + sizeof(struct nfs_fattr),
+		      GFP_KERNEL);
+	if (!mem)
+		goto out_bad;
+	fhandle = mem;
+	fattr = mem + sizeof(struct nfs_fh);
+
 	if (!inode) {
 		if (nfs_neg_need_reval(dir, dentry))
 			goto out_bad;
@@ -505,18 +530,19 @@
 	if (NFS_STALE(inode))
 		goto out_bad;
 
-	error = NFS_PROTO(dir)->lookup(dir, &dentry->d_name, &fhandle, &fattr);
+	error = NFS_PROTO(dir)->lookup(dir, &dentry->d_name, fhandle, fattr);
 	if (error)
 		goto out_bad;
-	if (memcmp(NFS_FH(inode), &fhandle, sizeof(struct nfs_fh))!= 0)
+	if (memcmp(NFS_FH(inode), fhandle, sizeof(struct nfs_fh))!= 0)
 		goto out_bad;
-	if ((error = nfs_refresh_inode(inode, &fattr)) != 0)
+	if ((error = nfs_refresh_inode(inode, fattr)) != 0)
 		goto out_bad;
 
 	nfs_renew_times(dentry);
  out_valid:
-	unlock_kernel();
-	return 1;
+ 	rc = 1;
+	goto out;
+
  out_bad:
 	NFS_CACHEINV(dir);
 	if (inode && S_ISDIR(inode->i_mode)) {
@@ -528,8 +554,14 @@
 		shrink_dcache_parent(dentry);
 	}
 	d_drop(dentry);
+	rc = 0;
+	/* fall through */
+
+out:
 	unlock_kernel();
-	return 0;
+	if (mem)
+		kfree(mem);
+	return rc;
 }
 
 /*
-
Quoted 6 lines

To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

-------------------------------------------------------
This sf.net email is sponsored by:ThinkGeek
Welcome to geek heaven.
http://thinkgeek.com/sf
_______________________________________________
NFS maillist - [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs

#2 lore raw
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

" " == Steve Dickson <[email protected]> writes:

> Also, not like there much choice in matter, but I wonder what > type of performance hit (if any) there will be by making these > routines call kmalloc()... lookups and readdirs are pretty > popular ops...

There are always alternatives...

If really this is a problem, how about slabifying the structs nfs_fattr and/or nfs_fh?

Also, since nfs_entry is only large due to the fh and fattr fields which are unused unless you have READDIRPLUS (in which case they are converted to pointers anyhow), how about kicking them out of nfs_entry altogether?

Cheers,
Trond

#3 lore raw
Quoted 7 lines · Trond Myklebust

There are always alternatives...

If really this is a problem, how about slabifying the structs nfs_fattr and/or nfs_fh?

Here is a patch that creates a new nfs_diropt_data slab that is used to allocate both the nfs_fh and nfs_fattrs structures.
The patch is based on the 2.4.23-pre5 kernel.

Quoted 6 lines

Also, since nfs_entry is only large due to the fh and fattr fields which are unused unless you have READDIRPLUS (in which case they are converted to pointers anyhow), how about kicking them out of nfs_entry altogether?

Instead of kicking them out, I just turned them into pointers...

Comments?

SteveD.

fs/nfs/inode.c renamed from fs/nfs/inode.c.orig +7 -0
@@ -1133,6 +1133,8 @@ extern int nfs_init_readpagecache(void);
 extern int nfs_destroy_readpagecache(void);
 extern int nfs_init_writepagecache(void);
 extern int nfs_destroy_writepagecache(void);
+extern int nfs_init_diroppagecache(void);
+extern void nfs_destroy_diroppagecache(void);
 
 /*
  * Initialize NFS
@@ -1153,6 +1155,10 @@ static int __init init_nfs_fs(void)
 	if (err)
 		return err;
 
+	err = nfs_init_diroppagecache();
+	if (err)
+		return err;
+
 #ifdef CONFIG_PROC_FS
 	rpc_proc_register(&nfs_rpcstat);
 #endif
@@ -1161,6 +1167,7 @@ static int __init init_nfs_fs(void)
 
 static void __exit exit_nfs_fs(void)
 {
+	nfs_destroy_diroppagecache();
 	nfs_destroy_writepagecache();
 	nfs_destroy_readpagecache();
 	nfs_destroy_nfspagecache();
fs/nfs/dir.c renamed from fs/nfs/dir.c.orig +97 -22

254 lines not on this page · raw message

fs/nfs/nfs3xdr.c renamed from fs/nfs/nfs3xdr.c.orig +3 -3
@@ -616,10 +616,10 @@ nfs3_decode_dirent(u32 *p, struct nfs_en
 	p = xdr_decode_hyper(p, &entry->cookie);
 
 	if (plus) {
-		p = xdr_decode_post_op_attr(p, &entry->fattr);
+		p = xdr_decode_post_op_attr(p, entry->fattr);
 		/* In fact, a post_op_fh3: */
 		if (*p++) {
-			p = xdr_decode_fhandle(p, &entry->fh);
+			p = xdr_decode_fhandle(p, entry->fh);
 			/* Ugh -- server reply was truncated */
 			if (p == NULL) {
 				dprintk("NFS: FH truncated\n");
@@ -629,7 +629,7 @@ nfs3_decode_dirent(u32 *p, struct nfs_en
 		} else {
 			/* If we don't get a file handle, the attrs
 			 * aren't worth a lot. */
-			entry->fattr.valid = 0;
+			entry->fattr->valid = 0;
 		}
 	}
 
include/linux/nfs_xdr.h renamed from include/linux/nfs_xdr.h.orig +2 -2
@@ -109,8 +109,8 @@ struct nfs_entry {
 	const char *		name;
 	unsigned int		len;
 	int			eof;
-	struct nfs_fh		fh;
-	struct nfs_fattr	fattr;
+	struct nfs_fh		*fh;
+	struct nfs_fattr	*fattr;
 };
 
 /*
#4 lore raw

UPDATED: Re: [NFS] Re: [PATCH v2] reduce NFS stack usage

While doing some testing with this patch I found
a hole in one of the return paths...

fs/nfs/dir.c 2003-09-29 20:36:46.000000000 -0400 renamed from fs/nfs/dir.c 2003-09-30 10:34:46.000000000 -0400 +1 -0
diff -u linux-2.4/fs/nfs/dir.c linux-2.4/fs/nfs/dir.c
@@ -572,7 +572,6 @@
       nfs_renew_times(dentry);
 out_valid:
       unlock_kernel();
+       nfs_dirop_free(fhandle, fattr);
       return 1;
 out_bad:
       NFS_CACHEINV(dir);

Attached is the updated patch...

SteveD.

fs/nfs/inode.c renamed from fs/nfs/inode.c.orig +7 -0
@@ -1133,6 +1133,8 @@ extern int nfs_init_readpagecache(void);
 extern int nfs_destroy_readpagecache(void);
 extern int nfs_init_writepagecache(void);
 extern int nfs_destroy_writepagecache(void);
+extern int nfs_init_diroppagecache(void);
+extern void nfs_destroy_diroppagecache(void);
 
 /*
  * Initialize NFS
@@ -1153,6 +1155,10 @@ static int __init init_nfs_fs(void)
 	if (err)
 		return err;
 
+	err = nfs_init_diroppagecache();
+	if (err)
+		return err;
+
 #ifdef CONFIG_PROC_FS
 	rpc_proc_register(&nfs_rpcstat);
 #endif
@@ -1161,6 +1167,7 @@ static int __init init_nfs_fs(void)
 
 static void __exit exit_nfs_fs(void)
 {
+	nfs_destroy_diroppagecache();
 	nfs_destroy_writepagecache();
 	nfs_destroy_readpagecache();
 	nfs_destroy_nfspagecache();
fs/nfs/dir.c renamed from fs/nfs/dir.c.orig +98 -22

260 lines not on this page · raw message

fs/nfs/nfs3xdr.c renamed from fs/nfs/nfs3xdr.c.orig +3 -3
@@ -616,10 +616,10 @@ nfs3_decode_dirent(u32 *p, struct nfs_en
 	p = xdr_decode_hyper(p, &entry->cookie);
 
 	if (plus) {
-		p = xdr_decode_post_op_attr(p, &entry->fattr);
+		p = xdr_decode_post_op_attr(p, entry->fattr);
 		/* In fact, a post_op_fh3: */
 		if (*p++) {
-			p = xdr_decode_fhandle(p, &entry->fh);
+			p = xdr_decode_fhandle(p, entry->fh);
 			/* Ugh -- server reply was truncated */
 			if (p == NULL) {
 				dprintk("NFS: FH truncated\n");
@@ -629,7 +629,7 @@ nfs3_decode_dirent(u32 *p, struct nfs_en
 		} else {
 			/* If we don't get a file handle, the attrs
 			 * aren't worth a lot. */
-			entry->fattr.valid = 0;
+			entry->fattr->valid = 0;
 		}
 	}
 
include/linux/nfs_xdr.h renamed from include/linux/nfs_xdr.h.orig +2 -2
@@ -109,8 +109,8 @@ struct nfs_entry {
 	const char *		name;
 	unsigned int		len;
 	int			eof;
-	struct nfs_fh		fh;
-	struct nfs_fattr	fattr;
+	struct nfs_fh		*fh;
+	struct nfs_fattr	*fattr;
 };
 
 /*

A message that is not in the archive (sent off-list or not yet received): [email protected]

#6 lore raw
Quoted 5 lines · Trond Myklebust

There are always alternatives...

If really this is a problem, how about slabifying the structs nfs_fattr and/or nfs_fh?

Why is allocating memory from a "private" slab is faster than using a general purpose one (i.e. GFP_KERNEL)? Something to do with lock contention?

Quoted 5 lines

Also, since nfs_entry is only large due to the fh and fattr fields which are unused unless you have READDIRPLUS (in which case they are converted to pointers anyhow), how about kicking them out of nfs_entry altogether?

That would take care of readdirs...

SteveD.

-------------------------------------------------------
This sf.net email is sponsored by:ThinkGeek
Welcome to geek heaven.
http://thinkgeek.com/sf
_______________________________________________
NFS maillist - [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs

#7 lore raw
In reply to Steve Dickson
Quoted 7 lines · Steve Dickson

Hey Jeff,

Question:
Why are only nfs_lookup_revalidate() and nfs_readdir() a problem and not the other 4 ops (like nfs_lookup())? Is the case only those two showed up in the stack overflow oops trace?

You guessed it... I'm sure other routines are problematic, but those were two that showed up in traces.

Quoted 4 lines

Also, not like there much choice in matter, but I wonder what type of performance hit (if any) there will be by making these routines call kmalloc()... lookups and readdirs are pretty popular ops...

Sure. I wasn't suggesting by any means my patch is the ultimate solution :) Creating a slab cache would be easy enough...

Jeff