statd simplified
Subsystems (4)
Hi all,
I've never really liked statd for a number of reasons (the following list is not exhaustive):
- 50% of all bug reports I get conerning "NFS file locking
broken" is due to people forgetting to start statd (the other half is people trying to do flock over NFS :)
- lockd having to do upcalls to user space is ugly
- user space doing RPC callbacks to the kernel is even uglier
- statd isn't really used by anything but lockd, so the
only reason for making things so complex is probably because the folks creating NFS locking _liked_ complexity.
So I sat down last week and dusted off some old ideas how to simplify NLM/NSM interaction quite a bit.
The approach I implemented was to get rid of all the SM_MON/SM_UNMON calls completely. If lockd wants a remote host to be "monitored", simply add it's IP address to /var/lib/nfs/sm. This used to be statd's job, but there's no reason why we cannot do that in the kernel.
In addition, lockd now understands a very limited subset of NSM
procedures: NULL, and SM_NOTIFY. If it receives an SM_NOTIFY message, it initiated reclaim for all locks we hold on that server (if any) and ditch all locks this client holds on our box (if any).
(As a special bonus, the new code actually initializes nsm_local_state from /var/lib/nfs/state, which we previously didn't).
That is almost all there is to statd. The only piece of the puzzle missing is a facility to send out NSM notifications when we reboot. That can be done in user space by running "rpc.statd -N" (notify-only mode) every time we boot.
Please find attached two kernel patches. The first is from Andreas Gruenbacher implementing several RPC programs sharing a single server socket (he wrote the patch for his NFS ACL implementation); the second is my kernel-statd patch.
Feedback welcome!
Cheers,
Olaf
Signature · 3 lines
Olaf Kirch | Stop wasting entropy - start using predictable [email protected] | tempfile names today! ---------------+
Support multiple program numbers on one RPC transport
The NFS and NFSACL programs run on the same RPC transport. This patch adds support for this by changing svc_program into a chained list of programs instead of a single program (on the server side).
On the client side, the same RPC transport can now be shared among multiple RPC clients, so reference count the transport to determine when it can be destroyed.
Andreas Gruenbacher <[email protected]>, SuSE Labs
fs/nfsd/nfsproc.c +2 -0
@@ -587,6 +587,8 @@ nfserrno (int errno) { nfserr_stale, -ESTALE }, { nfserr_dropit, -EAGAIN }, { nfserr_dropit, -ENOMEM }, + { nfserr_notsupp, -ENOTSUPP }, + { nfserr_notsupp, -EOPNOTSUPP }, { -1, -EIO } };int i;
net/sunrpc/svc.c +5 -3
@@ -324,8 +324,10 @@ svc_process(struct svc_serv *serv, struc goto dropit; } - progp = serv->sv_program; -if (prog != progp->pg_prog) +for (progp = serv->sv_program; progp; progp = progp->pg_next) +if (prog == progp->pg_prog) +break ; +if (progp ==NULL ) goto err_bad_prog; if (vers >= progp->pg_nvers ||
@@ -438,7 +440,7 @@ err_bad_auth: err_bad_prog:#ifdef RPC_PARANOIA -if (prog !=100227 || progp->pg_prog !=100003 ) +if (prog !=100227 || serv->sv_program->pg_prog !=100003 )printk ("svc: unknown program %d (me %d)\n" , prog, progp->pg_prog);/* else it is just a Solaris client seeing if ACLs are supported */ #endif
include/linux/sunrpc/svc.h +2 -1
@@ -231,9 +231,10 @@ struct svc_deferred_req { };/* -* RPC program +* List of RPC programs on the same transport endpoint */ struct svc_program { +struct svc_program * pg_next;/* other programs (same xprt) */ u32 pg_prog;/* program number */ unsigned int pg_lovers;/* lowest version */ unsigned int pg_hivers;/* lowest version */
fs/Kconfig +4 -0
@@ -1466,6 +1466,10 @@ config ROOT_NFS config LOCKD tristate +config STATD + bool "Use kernel statd implementation" + depends on LOCKD && EXPERIMENTAL + config LOCKD_V4 bool depends on NFSD_V3 || NFS_V3
fs/buffer.c +1 -0
@@ -242,6 +242,7 @@return sync_blockdev (sb->s_bdev); } +EXPORT_SYMBOL (fsync_super);/* * Write out and wait upon all dirty data associated with this
fs/lockd/Makefile +7 -1
@@ -5,6 +5,12 @@ obj-$(CONFIG_LOCKD) += lockd.o lockd-objs-y := clntlock.o clntproc.o host.o svc.o svclock.o svcshare.o \ - svcproc.o svcsubs.o mon.o xdr.o lockd_syms.o + svcproc.o svcsubs.o xdr.o lockd_syms.o +ifeq ($(CONFIG_STATD),y) +lockd-objs-y += statd.o +else +lockd-objs-y += mon.o +endif + lockd-objs-$(CONFIG_LOCKD_V4) += xdr4.o svc4proc.o lockd-objs := $(lockd-objs-y)
fs/lockd/host.c +50 -0
56 lines not on this page · raw message
fs/lockd/mon.c +11 -1
@@ -3,6 +3,10 @@* * The kernel statd client. * +* When using the kernel statd implementation, none of the +* stuff inside this file is used. +* Instead look at statd.c +* * Copyright (C) 1996, Olaf Kirch <[email protected]> */
@@ -15,6 +19,9 @@#include <linux/lockd/sm_inter.h> + +#ifndef CONFIG_STATD +#define NLMDBG_FACILITY NLMDBG_MONITORstatic struct rpc_clnt *nsm_create (void );
@@ -22,7 +29,8 @@ static struct rpc_clnt * nsm_create(voidextern struct rpc_program nsm_program;/* -* Local NSM state +* Local NSM state. +* This should really be initialized somehow. */ u32 nsm_local_state;
@@ -246,3 +254,5 @@ struct rpc_program nsm_program = { .version = nsm_version, .stats = &nsm_stats }; + +#endif
fs/lockd/statd.c +278 -0
278 lines not on this page · raw message
fs/lockd/svc.c +38 -0
@@ -34,6 +34,7 @@#include <linux/sunrpc/svc.h> #include <linux/sunrpc/svcsock.h> #include <linux/lockd/lockd.h> +#include <linux/lockd/sm_inter.h> #include <linux/nfs.h> #define NLMDBG_FACILITY NLMDBG_SVC
@@ -115,6 +116,11 @@ lockd(struct svc_rqst *rqstp)daemonize ("lockd" ); +#ifdef CONFIG_STATD +/* Set up statd */ +nsm_init (); +#endif +/* Process request with signals blocked, but allow SIGKILL. */ allow_signal (SIGKILL );
@@ -439,6 +445,37 @@ static void __exit exit_nlm(void)module_init (init_nlm);module_exit (exit_nlm); +#ifdef CONFIG_STATD +/* +* Define NSM program and procedures +*/ +static struct svc_version nsmsvc_version1 = { + .vs_vers =1 , + .vs_nproc =5 , + .vs_proc = nsmsvc_procedures, + .vs_xdrsize =SMSVC_XDRSIZE , +}; +static struct svc_version * nsmsvc_version[] = { + [1 ] = &nsmsvc_version1, +}; + +static struct svc_stat nsmsvc_stats; + +#define SM_NRVERS (sizeof(nsmsvc_version)/sizeof(nsmsvc_version[0])) +static struct svc_program nsmsvc_program = { + .pg_prog =SM_PROGRAM ,/* program number */ + .pg_nvers =SM_NRVERS ,/* number of entries in nlmsvc_version */ + .pg_vers = nsmsvc_version,/* version table */ + .pg_name ="statd" ,/* service name */ + .pg_class ="nfsd" ,/* share authentication with nfsd */ + .pg_stats = &nsmsvc_stats,/* stats table */ +}; + +#define nsmsvc_program_p &nsmsvc_program +#else +#define nsmsvc_program_p NULL +#endif +/* * Define NLM program and procedures */
@@ -474,6 +511,7 @@ static struct svc_stat nlmsvc_stats;#define NLM_NRVERS (sizeof(nlmsvc_version)/sizeof(nlmsvc_version[0]))struct svc_program nlmsvc_program = { + .pg_next = nsmsvc_program_p, .pg_prog =NLM_PROGRAM ,/* program number */ .pg_nvers =NLM_NRVERS ,/* number of entries in nlmsvc_version */ .pg_vers = nlmsvc_version,/* version table */
include/linux/lockd/lockd.h +6 -1
@@ -48,7 +48,8 @@ struct nlm_host { h_server : 1, /* server side, not client side */ h_inuse : 1, h_killed : 1, - h_monitored : 1; + h_monitored : 1, + h_rebooted : 1;wait_queue_head_t h_gracewait;/* wait while reclaiming */ u32 h_state;/* pseudo-state counter */ u32 h_nsmstate;/* true remote NSM state */
@@ -121,6 +122,9 @@ extern struct svc_procedure nlmsvc_proce#ifdef CONFIG_LOCKD_V4 extern struct svc_procedure nlmsvc_procedures4[];#endif +#ifdef CONFIG_STATD +extern struct svc_procedure nsmsvc_procedures[]; +#endif extern int nlmsvc_grace_period;extern unsigned long nlmsvc_timeout;
@@ -150,6 +154,7 @@ struct nlm_host * nlm_get_host(struct nlvoid nlm_release_host (struct nlm_host *);void nlm_shutdown_hosts (void );extern struct nlm_host *nlm_find_client (void ); +extern void nlm_host_rebooted (struct sockaddr_in *,u32 );/*
include/linux/lockd/sm_inter.h +3 -0
@@ -19,6 +19,7 @@#define SM_NOTIFY 6#define SM_MAXSTRLEN 1024 +#define SMSVC_XDRSIZE sizeof(struct nsm_args)/* * Arguments for all calls to statd
@@ -29,6 +30,7 @@ struct nsm_args {u32 vers;u32 proc;u32 proto;/* protocol (udp/tcp) plus server/client flag */ +u32 state;/* in NOTIFY calls */ };/*
@@ -39,6 +41,7 @@ struct nsm_res {u32 state; }; +extern int nsm_init (void );int nsm_monitor (struct nlm_host *);int nsm_unmonitor (struct nlm_host *);extern u32 nsm_local_state;
net/sunrpc/svc.c +17 -12
@@ -221,22 +221,27 @@ svc_register(struct svc_serv *serv, int progp = serv->sv_program; -dprintk ("RPC: svc_register(%s, %s, %d)\n" , - progp->pg_name, proto ==IPPROTO_UDP ?"udp" :"tcp" , port); -if (!port)clear_thread_flag (TIF_SIGPENDING ); -for (i =0 ; i < progp->pg_nvers; i++) { -if (progp->pg_vers[i] ==NULL ) -continue ; - error =rpc_register (progp->pg_prog, i, proto, port, &dummy); -if (error <0 ) -break ; -if (port && !dummy) { - error = -EACCES ; -break ; +while (progp) { +dprintk ("RPC: svc_register(%s, %s, %d)\n" , + progp->pg_name, + proto ==IPPROTO_UDP ?"udp" :"tcp" , + port); + +for (i =0 ; i < progp->pg_nvers; i++) { +if (progp->pg_vers[i] ==NULL ) +continue ; + error =rpc_register (progp->pg_prog, i, proto, port, &dummy); +if (error <0 ) +break ; +if (port && !dummy) { + error = -EACCES ; +break ; + } } + progp = progp->pg_next; }if (!port) {
Quoted 4 lines · Olaf Kirch
So I sat down last week and dusted off some old ideas how to simplify NLM/NSM interaction quite a bit.
Putting statd in the kernel I think a good idea... Probably the only reason it has not been done, is the fact, managing the state file from the kernel is a real pain...
Quoted 6 lines
The approach I implemented was to get rid of all the SM_MON/SM_UNMON calls completely. If lockd wants a remote host to be "monitored", simply add it's IP address to
/var/lib/nfs/sm. This used to be statd's job, but there's no reason why we cannot do that in the kernel.
Letting /var/lib/nfs/sm grow uncontrollably (i.e. not using SM_UNMON to clean it up) may not be a good idea... Since users can't go in and clean in up by hand (since they could be removing live state), there probably should be some way to cleaned up (and up to date)
Quoted 9 lines
In addition, lockd now understands a very limited subset of NSM
procedures: NULL, andSM_NOTIFY. If it receives anSM_NOTIFYmessage, it initiated reclaim for all locks we hold on that server (if any) and ditch all locks this client holds on our box (if any).(As a special bonus, the new code actually initializes
nsm_local_statefrom/var/lib/nfs/state, which we previously didn't).
Currently when statd can not open the state file (in change_state()), it dies which in turn causes lock requests to fail because that can't be monitored. In your kernel implementation, this failure seems to be ignored. Since the failure would probably do to the sm directory not existing, which also means state file will not able to be created, it might make sense to fail the loading of lockd when this happens.
Also I notice that the new nsm_monitor() will only fail with ENOMEM, regardless if it was able to save state... If it can not save state, shouldn't it fail?
SteveD.
-------------------------------------------------------
The SF.Net email is sponsored by EclipseCon 2004 Premiere Conference on Open Tools Development and Integration See the breadth of Eclipse activity. February 3-5 in Anaheim, CA.
http://www.eclipsecon.org/osdn
_______________________________________________
NFS maillist - [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs
Hi,
Quoted 5 lines · Steve Dickson
Letting
/var/lib/nfs/smgrow uncontrollably (i.e. not usingSM_UNMONto clean it up) may not be a good idea... Since users can't go in and clean in up by hand
(since they could be removing live state), there probably should be some way to cleaned up (and up to date)
Okay, I could add that. Calling unlink from kernel space is a bit of a nuisance, but I'll try :)
Quoted 7 lines
Currently when statd can not open the state file (in
change_state()), it dies
which in turn causes lock requests to fail because that can't be monitored. In your kernel implementation, this failure seems to be ignored. Since the failure would probably do to the sm directory not existing, which also means state file will not able to be created, it might make sense to fail the loading of lockd when this happens.
Agreed. Kernel statd just reads the file, however, it doesn't update the state. That's done by rpc.statd -N.
I'll submit an updated patch later today or tomorrow.
Olaf
Signature · 13 lines
Olaf Kirch | Stop wasting entropy - start using predictable [email protected] | tempfile names today! ---------------+ ------------------------------------------------------- The SF.Net email is sponsored by EclipseCon 2004 Premiere Conference on Open Tools Development and Integration See the breadth of Eclipse activity. February 3-5 in Anaheim, CA. http://www.eclipsecon.org/osdn _______________________________________________ NFS maillist - [email protected] https://lists.sourceforge.net/lists/listinfo/nfs
Hello,
Quoted 10 lines · Olaf Kirch
Quoted 6 lines · Steve Dickson
Letting
/var/lib/nfs/smgrow uncontrollably (i.e. not usingSM_UNMONto clean it up) may not be a good idea... Since users can't go in and clean in up by hand (since they could be removing live state), there probably should be some way to cleaned up (and up to date)
Quoted 4 lines
Okay, I could add that. Calling unlink from kernel space is a bit of a nuisance, but I'll try :)
Maybe the user level daemon could be used to do this.... When an SM_UNMON comes in, the kernel could clean up its state and then forward the message to the daemon that would cause the file to be removed...
SteveD.
-------------------------------------------------------
The SF.Net email is sponsored by EclipseCon 2004 Premiere Conference on Open Tools Development and Integration See the breadth of Eclipse activity. February 3-5 in Anaheim, CA.
http://www.eclipsecon.org/osdn
_______________________________________________
NFS maillist - [email protected]
https://lists.sourceforge.net/lists/listinfo/nfs