NFS: Zeroing NFS and kNFSD stats

Here is a patch I've been using for quite a while now that will zero out the nfs stats on both the server and client side.

There are two interface: nfsstat and /proc. The nfstat interface looks like (from the manpage):
-z Zeros out all or some of the statistics. Typical uses would be:
nfsstat -z - zeros all statistics nfsstat -zc - zeros only client statistics nfsstat -zs - zeros only server statistics nfsstat -zr - zeros only RPC statistics nfsstat -zn - zeros only NFS call statistics

Note: the nfs-utils in the Fedora Core 2/3 release already have this support.

The /proc interface look something like:

echo nfs > /proc/net/rpc/nfs[d] # zeros calls stats echo rpc > /proc/net/rpc/nfs[d] # zeros rpc stats echo net > /proc/net/rpc/nfs[d] # zeros connection stats

echo rc  > /proc/net/rpc/nfsd   # zeros server cache stats
echo fh  > /proc/net/rpc/nfsd   # zeros server fh stats

Again this is a patch I've had kicking around for sometime so its pretty well tested... and I'm a bit dependent on it in the sense its annoying when I go to zero the stats and I can't because I forgot to apply the patch...

I'm hopeful that other people will also find this patch useful as well and it will make its way into an upstream kernel in the very near future....

SteveD.

@@ -48,21 +48,34 @@ void			rpc_modcount(struct inode *, int)
 #ifdef CONFIG_PROC_FS
 struct proc_dir_entry *	rpc_proc_register(struct rpc_stat *);
 void			rpc_proc_unregister(const char *);
-void			rpc_proc_zero(struct rpc_program *);
 struct proc_dir_entry *	svc_proc_register(struct svc_stat *,
 					  struct file_operations *);
 void			svc_proc_unregister(const char *);
 
 void			svc_seq_show(struct seq_file *,
 				     const struct svc_stat *);
+void			svc_seq_zero(struct seq_file *,
+				     struct svc_stat *, unsigned int);
 
 extern struct proc_dir_entry	*proc_net_rpc;
+/*
+ * Bits used to zero out status
+ */
+#define PRNT_CALLS  0x0001
+#define PRNT_RPC    0x0002
+#define PRNT_NET    0x0004
+#define PRNT_FH     0x0008
+#define PRNT_RC     0x0010
+#define PRNT_IO     0x0020
+#define PRNT_ALL    0xffff
+
+unsigned int rpc_proc_getval(char *, int , const char *, size_t);
 
 #else
 
 static inline struct proc_dir_entry *rpc_proc_register(struct rpc_stat *s) { return NULL; }
 static inline void rpc_proc_unregister(const char *p) {}
-static inline void rpc_proc_zero(struct rpc_program *p) {}
+static inline rpc_proc_getval(char *, int , const char *, unsigned long) {}
 
 static inline struct proc_dir_entry *svc_proc_register(struct svc_stat *s,
 						       struct file_operations *f) { return NULL; }

@@ -70,6 +83,8 @@ static inline void svc_proc_unregister(c
 
 static inline void svc_seq_show(struct seq_file *seq,
 				const struct svc_stat *st) {}
+static inline void svc_seq_zero(struct seq_file *seq,
+				const struct svc_stat *st, unsigned int opt) {}
 
 #define proc_net_rpc NULL
 

@@ -74,6 +74,42 @@ static int nfsd_proc_show(struct seq_fil
 
 	return 0;
 }
+#define HEX_DIGITS 8
+
+static ssize_t
+nfsd_proc_zero(struct file *file, const char *buffer, 
+	size_t count, loff_t *data)
+{
+	struct seq_file *seq = (struct seq_file *)file->private_data;
+	char hexnum [HEX_DIGITS];
+	unsigned int opt_prt;
+
+	opt_prt = rpc_proc_getval(hexnum, HEX_DIGITS, buffer, count);
+	if (opt_prt < 0) 
+		return opt_prt;
+
+	if (opt_prt & PRNT_RC) {
+		nfsdstats.rchits = 0;
+		nfsdstats.rcmisses = 0;
+		nfsdstats.rcnocache = 0;
+	}
+	if (opt_prt & PRNT_FH) {
+		nfsdstats.fh_stale = 0;
+		nfsdstats.fh_lookup = 0;
+		nfsdstats.fh_anon = 0;
+		nfsdstats.fh_nocache_dir = 0;
+		nfsdstats.fh_nocache_nondir = 0;
+	}
+	if (opt_prt & PRNT_IO) {
+		nfsdstats.io_read = 0;
+		nfsdstats.io_write = 0;
+	}
+
+	/* zero my rpc info */
+	svc_seq_zero(seq, &nfsd_svcstats, opt_prt);
+
+	return opt_prt;
+}
 
 static int nfsd_proc_open(struct inode *inode, struct file *file)
 {

@@ -84,6 +120,7 @@ static struct file_operations nfsd_proc_
 	.owner = THIS_MODULE,
 	.open = nfsd_proc_open,
 	.read  = seq_read,
+	.write  = nfsd_proc_zero,
 	.llseek = seq_lseek,
 	.release = single_release,
 };

@@ -94,9 +94,11 @@ EXPORT_SYMBOL(auth_domain_lookup);
 #ifdef CONFIG_PROC_FS
 EXPORT_SYMBOL(rpc_proc_register);
 EXPORT_SYMBOL(rpc_proc_unregister);
+EXPORT_SYMBOL_GPL(rpc_proc_getval);
 EXPORT_SYMBOL(svc_proc_register);
 EXPORT_SYMBOL(svc_proc_unregister);
 EXPORT_SYMBOL(svc_seq_show);
+EXPORT_SYMBOL_GPL(svc_seq_zero);
 #endif
 
 /* caching... */

What is it useful for?

--Bruce Fields

-------------------------------------------------------
This SF.Net email sponsored by Black Hat Briefings & Training. Attend Black Hat Briefings & Training, Las Vegas July 24-29 - digital self defense, top technical experts, no vendor pitches, unmatched networking opportunities. Visit www.blackhat.com
_______________________________________________
NFS maillist - [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs

A few things... One to see if how traffic is flowing... sure you can look a the numbers go up but to be able to start from zero gives you a better idea on how well things are flowing. I also use it for testing patchs that deal with caching or RPC changes to see what impact they have on OTW traffic... Another use it to see what opts are really being tested... Starting up a test run, zeroing out the stats give you a very clear picture of what exactly is going on... There probably a few more uses that are just not coming to me right now... Its one of those tools that I keep finding ways to use... :)

SteveD.

-------------------------------------------------------
This SF.Net email sponsored by Black Hat Briefings & Training.
Attend Black Hat Briefings & Training, Las Vegas July 24-29 - digital self defense, top technical experts, no vendor pitches, unmatched networking opportunities. Visit www.blackhat.com
_______________________________________________
NFS maillist - [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs

But you could exactly the same thing by recording the values at the beginning of the test run and then subtracting. In practice this is likely to be annoying, so you'd want to write utilities that did this for you, like say

checkpoint_stats stats.txt
display_stats --since stats.txt

That'd do what you want, right?

--Bruce Fields

-------------------------------------------------------
This SF.Net email sponsored by Black Hat Briefings & Training. Attend Black Hat Briefings & Training, Las Vegas July 24-29 - digital self defense, top technical experts, no vendor pitches, unmatched networking opportunities. Visit www.blackhat.com
_______________________________________________
NFS maillist - [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs

The problem I have, by the way, is that zeroing is going to cause confusing problems whenever there are two simultaneous attempts to collect statistics.

For example, imagine that your NFS server has a persistent performance-monitoring daemon that checks the statistics every now and then and emails the adminstrator a summary once a week.

Then imagine that one day you have a problem with the server, so you log in and do a quick check of some statistics, zeroing them in the process. Now the daemon has no more hope of getting its numbers right. Murphy's law being what it is, you and the adminstrator that gets the summary email are probably different people, and by the time he/she gets the very odd-looking end-of-week report you've forgotten that you zeroed the statistics once. Head-scratching ensues.

Writing a script to do the subtraction is just as simple and avoids any such problems.

--b.

-------------------------------------------------------
This SF.Net email sponsored by Black Hat Briefings & Training. Attend Black Hat Briefings & Training, Las Vegas July 24-29 - digital self defense, top technical experts, no vendor pitches, unmatched networking opportunities. Visit www.blackhat.com
_______________________________________________
NFS maillist - [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs

IMHO, its much simpler, easier and less error prone to zero them out than to make nfsstat take samples then calculate the differences... I just think that a bit over nfsstat's head...

Hang on.... Lets keep this in prospective.... Stats that calculate bit rates,latencies and such (Like the ones in Chuck's per-mount patchs) should not be zeroed out, since it simply does not make sense, due to the collection issues you've outline... but I contend that the current nfsd/rpc stats simply aren't these type of stats... these are simple activity counters that let you know what (if anything) is happening... I think it would be a very difficult task to try to used these counts in any long term analysis... I just don't think they recored the right type of information for that....

but... I do think allowing them to be reinitialized (maybe that's better term 8-)) makes them much more useful than they currently are... TBL, when was the last time anybody looked at these stats because they are basically unmanageable, having the ability to zero them out, make them much more useful (imho)....

SteveD.

-------------------------------------------------------
This SF.Net email sponsored by Black Hat Briefings & Training.
Attend Black Hat Briefings & Training, Las Vegas July 24-29 - digital self defense, top technical experts, no vendor pitches, unmatched networking opportunities. Visit www.blackhat.com
_______________________________________________
NFS maillist - [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs

Example 1:
nfsstat -z
nfsstat
Example 2:

	nfsstat --checkpoint foo
	nfsstat --since foo

Example 1 is indeed a little simpler to use, but not much.

If you really wanted an interface like that in case 1, you could do that by storing the checkpointed stats in some default location.

If the typical use is just logging in and collecting a few seconds of statistics then the easiest thing might be something like

	nfsstat -z
		Collecting statistics.... Hit ^C for results
	^C
	Server rpc stats:
	...etc.

I'm not sure what you mean. It certainly shouldn't be hard to implement.

It's not obvious to me why they couldn't be used over long periods. But even if the statistics-gathering daemon isn't a good example, I don't see why there couldn't be other cases where more than one process might happen to be trying to collect statistics at the same time.

--b.

-------------------------------------------------------
This SF.Net email is sponsored by BEA Weblogic Workshop FREE Java Enterprise J2EE developer tools!
Get your free copy of BEA WebLogic Workshop 8.1 today.
http://ads.osdn.com/?ad_id=4721&alloc_id=10040&op=click
_______________________________________________
NFS maillist - [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs

Here is something that Even Geller, the 14 year old intern that we have here at the lab, whipped up for our own purposes. Not everything is done exactly the way that I would have done it but I was pretty impressed for an afternoon's work by a 14 year old. I'm sure he will accept patches. ;-)

-ben

On Tue, Jul 13, 2004 at 02:01:08PM -0400, J. Bruce Fields alleged:

It's about choice, isn't it? Some sites may want to maintain the same numbers over time, other sites might want to zero them.

If someone zeros the numbers and messes up something else, that's the admin's perogative to break things.

Garrick Staples, Linux/HPCC Administrator
University of Southern California

What choice? You can do everything you need with the current kernel interface.

Maybe, but this gives the administrator one more way to break things without giving them any real new functionality. On balance that seems a poor tradeoff.

--b.

-------------------------------------------------------
This SF.Net email is sponsored by BEA Weblogic Workshop FREE Java Enterprise J2EE developer tools!
Get your free copy of BEA WebLogic Workshop 8.1 today.
http://ads.osdn.com/?ad_id=4721&alloc_id=10040&op=click
_______________________________________________
NFS maillist - [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs

But it sucks. I had folks run something like 12-15 performance measurements and take the NFS stats, and it was a huge mess to sort through. Telling folks "use nfsstat -z to wipe stats before each command" is much easier. Besides, -z is sort of a standard option, so we might as well implement it.

Olaf

Olaf Kirch     |  The Hardware Gods hate me.
[email protected]   |
---------------+ 


-------------------------------------------------------
This SF.Net email is sponsored by OSTG. Have you noticed the changes on
Linux.com, ITManagersJournal and NewsForge in the past few weeks? Now,
one more big change to announce. We are now OSTG- Open Source Technology
Group. Come see the changes on the new OSTG site. www.ostg.com
_______________________________________________
NFS maillist  -  [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs

Well, like I say, we could implement "nfsstat -z" by checkpointing as above but just writing to a file at some default path, though admittedly the choice of path might not be obvious.

Having nfsstat just hang around till it's signaled (like tcpdump does) would also be easy.

Well, OK, I suppose that's an argument for it.--Bruce Fields

-------------------------------------------------------
This SF.Net email is sponsored by OSTG. Have you noticed the changes on Linux.com, ITManagersJournal and NewsForge in the past few weeks? Now, one more big change to announce. We are now OSTG- Open Source Technology Group. Come see the changes on the new OSTG site. www.ostg.com
_______________________________________________
NFS maillist - [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs

FWIW, futzing with zeroing or checkpointing should not be necessary on Altix systems because they ship with PCP, a stats gathering package which samples the NFS stats (among many others) and does rate conversion.

http://oss.sgi.com/projects/pcp/

IRIX' nfsstat has a -C option which uses curses to display every 1 sec the rates of various NFS stats, until interrupted. It's a bit like "watch nfsstat" but shows rate converted numbers instead of changing counters. Here's an example screen, for a machine getting 30 RPC calls/sec:

4: Server snort Aug 5 15:21:21 D: Delta/second

Server RPC:

calls      badcalls   nullrecv   badlen     xdrcall    duphits    dupage
30         0          0          0          0          0          40086.17

Server NFS V3:

calls        badcalls
30           0
null         getattr      setattr      lookup       access       readlink
1  0%        10 33%       1  2%        16 52%       1  0%        0  0%
read         write        create       mkdir        symlink      mknod
2  4%        1  3%        0  0%        0  0%        0  0%        0  0%
remove       rmdir        rename       link         readdir      readdir+
0  0%        0  0%        0  0%        0  0%        0  0%        0  0%
fsstat       fsinfo       pathconf     commit
1  1%        0  0%        0  0%        1  2%  

1: Client[V2] 2: Server[V2] 3: Client[V3] 4: Server[V3] 5: Daemons DZR:mode

Something like that might be more useful than the -z option.

Greg.

Greg Banks, R&D Software Engineer, SGI Australian Software Group.
I don't speak for SGI.


-------------------------------------------------------
This SF.Net email is sponsored by OSTG. Have you noticed the changes on
Linux.com, ITManagersJournal and NewsForge in the past few weeks? Now,
one more big change to announce. We are now OSTG- Open Source Technology
Group. Come see the changes on the new OSTG site. www.ostg.com
_______________________________________________
NFS maillist  -  [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs