un-deprecate nfsdcld
Subsystems (2)
cover un-deprecate nfsdcld
When nfsdcld was released, it was quickly deprecated in favor of the nfsdcltrack usermodehelper, so as to not require another running daemon. That prevents NFSv4 clients from reclaiming locks from nfsd's running in containers, since neither nfsdcltrack nor the legacy client tracking code work in containers. These patches un-deprecate the use of nfsdcld for NFSv4 client tracking.
These patches are intended to go alongside some nfs-utils patches that introduce an enhancement that allows nfsd to "slurp" up the client records during client tracking initialization and store them internally in hash table. This enables nfsd to check whether an NFSv4 client is allowed to reclaim without having to do an upcall to nfsdcld. It also allows nfsd to decide to end the v4 grace period early if the number of RECLAIM_COMPLETE operations it has received from "known" clients is equal to the number of entries in the hash table. It also allows nfsd to skip the v4 grace period altogether if it knows there are no clients allowed to reclaim.
There is a fallback to allow nfsd to continue to work with older nfsdcld daemons in the event that any are out in the wild (unlikely). Everything should work fine except nfsd will not be able to exit the grace period early or skip the grace period altogether.
Scott Mayhew (4):
nfsd: fix a warning in __cld_pipe_upcall()
nfsd: make nfs4_client_reclaim use an xdr_netobj instead of a fixed
char array
nfsd: un-deprecate nfsdcld
nfsd: keep a tally of RECLAIM_COMPLETE operations when using nfsdcld
fs/nfsd/netns.h | 3 + fs/nfsd/nfs4recover.c | 326 ++++++++++++++++++++++++++++++---- fs/nfsd/nfs4state.c | 82 +++++++-- fs/nfsd/nfsctl.c | 1 + fs/nfsd/state.h | 8 +- include/uapi/linux/nfsd/cld.h | 1 + 6 files changed, 372 insertions(+), 49 deletions(-)
Signature · 1 line
2.17.1
RFC PATCH 1/4 nfsd: fix a warning in __cld_pipe_upcall()
__cld_pipe_upcall() emits a "do not call blocking ops when !TASK_RUNNING" warning due to the dput() call in rpc_queue_upcall(). Fix it by using a completion instead of hand coding the wait.
--- fs/nfsd/nfs4recover.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-)
fs/nfsd/nfs4recover.c +6 -11
@@ -662,7 +662,7 @@ struct cld_net {struct cld_upcall {struct list_head cu_list;struct cld_net *cu_net; -struct task_struct *cu_task; +struct completion cu_done;struct cld_msg cu_msg; };
@@ -671,23 +671,18 @@ __cld_pipe_upcall(struct rpc_pipe *pipe, struct cld_msg *cmsg) {int ret;struct rpc_pipe_msg msg; + struct cld_upcall *cup = container_of(cmsg, struct cld_upcall, cu_msg);memset (&msg,0 ,sizeof (msg)); msg.data = cmsg; msg.len =sizeof (*cmsg); -/* -* Set task state before we queue the upcall. That prevents -* wake_up_process in the downcall from racing with schedule. -*/ -set_current_state (TASK_UNINTERRUPTIBLE ); ret =rpc_queue_upcall (pipe, &msg);if (ret <0 ) { -set_current_state (TASK_RUNNING ); goto out; } -schedule (); +wait_for_completion (&cup->cu_done);if (msg.errno <0 ) ret = msg.errno;
@@ -754,7 +749,7 @@ cld_pipe_downcall(struct file *filp, const char __user *src, size_t mlen)if (copy_from_user (&cup->cu_msg, src, mlen) !=0 )return -EFAULT ; -wake_up_process (cup->cu_task); +complete (&cup->cu_done);return mlen; }
@@ -769,7 +764,7 @@ cld_pipe_destroy_msg(struct rpc_pipe_msg *msg)if (msg->errno >=0 )return ; -wake_up_process (cup->cu_task); +complete (&cup->cu_done); }static const struct rpc_pipe_ops cld_upcall_ops = {
@@ -900,7 +895,7 @@ alloc_cld_upcall(struct cld_net *cn) goto restart_search; } } - new->cu_task = current; +init_completion (&new->cu_done); new->cu_msg.cm_vers =CLD_UPCALL_VERSION ;put_unaligned (cn->cn_xid++, &new->cu_msg.cm_xid); new->cu_net = cn;
Signature · 1 line
2.17.1
Re: [PATCH RFC 1/4] nfsd: fix a warning in __cld_pipe_upcall()
This looks fine, applying now for 4.21.--b.
Quoted 77 lines · Scott Mayhew
__cld_pipe_upcall()emits a "do not call blocking ops when !TASK_RUNNING" warning due to thedput()call inrpc_queue_upcall(). Fix it by using a completion instead of hand coding the wait.Signed-off-by Scott Mayhew <[email protected]>--- fs/nfsd/nfs4recover.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-)fs/nfsd/nfs4recover.c +6
-11@@ -662,7 +662,7 @@ struct cld_net {struct cld_upcall {struct list_head cu_list;struct cld_net *cu_net; -struct task_struct *cu_task; +struct completion cu_done;struct cld_msg cu_msg; };@@ -671,23 +671,18 @@ __cld_pipe_upcall(struct rpc_pipe *pipe, struct cld_msg *cmsg) {int ret;struct rpc_pipe_msg msg; + struct cld_upcall *cup = container_of(cmsg, struct cld_upcall, cu_msg);memset (&msg,0 ,sizeof (msg)); msg.data = cmsg; msg.len =sizeof (*cmsg); -/* -* Set task state before we queue the upcall. That prevents -* wake_up_process in the downcall from racing with schedule. -*/ -set_current_state (TASK_UNINTERRUPTIBLE ); ret =rpc_queue_upcall (pipe, &msg);if (ret <0 ) { -set_current_state (TASK_RUNNING ); goto out; } -schedule (); +wait_for_completion (&cup->cu_done);if (msg.errno <0 ) ret = msg.errno;@@ -754,7 +749,7 @@ cld_pipe_downcall(struct file *filp, const char __user *src, size_t mlen)if (copy_from_user (&cup->cu_msg, src, mlen) !=0 )return -EFAULT ; -wake_up_process (cup->cu_task); +complete (&cup->cu_done);return mlen; }@@ -769,7 +764,7 @@ cld_pipe_destroy_msg(struct rpc_pipe_msg *msg)if (msg->errno >=0 )return ; -wake_up_process (cup->cu_task); +complete (&cup->cu_done); }static const struct rpc_pipe_ops cld_upcall_ops = {@@ -900,7 +895,7 @@ alloc_cld_upcall(struct cld_net *cn) goto restart_search; } } - new->cu_task = current; +init_completion (&new->cu_done); new->cu_msg.cm_vers =CLD_UPCALL_VERSION ;put_unaligned (cn->cn_xid++, &new->cu_msg.cm_xid); new->cu_net = cn;Signature · 1 line
2.17.1
RFC PATCH 2/4 nfsd: make nfs4_client_reclaim use an xdr_netobj instead of a fixed char array
In reply to Scott MayhewThis will allow the reclaim_str_hashtbl to store either the recovery directory names used by the legacy client tracking code or the full client strings used by the nfsdcld client tracking code.
--- fs/nfsd/nfs4recover.c | 79 +++++++++++++++++++++++++++++++++++++------ fs/nfsd/nfs4state.c | 30 ++++++++-------- fs/nfsd/state.h | 8 ++--- 3 files changed, 87 insertions(+), 30 deletions(-)
fs/nfsd/nfs4recover.c +68 -11
@@ -178,6 +178,7 @@ nfsd4_create_clid_dir(struct nfs4_client *clp)struct nfs4_client_reclaim *crp;int status;struct nfsd_net *nn =net_generic (clp->net, nfsd_net_id); +struct xdr_netobj name;if (test_and_set_bit (NFSD4_CLIENT_STABLE , &clp->cl_flags))return ;
@@ -222,9 +223,18 @@ nfsd4_create_clid_dir(struct nfs4_client *clp)inode_unlock (d_inode (dir));if (status ==0 ) {if (nn->in_grace) { - crp =nfs4_client_to_reclaim (dname, nn); -if (crp) - crp->cr_clp = clp; + name.data =kmemdup (dname,HEXDIR_LEN ,GFP_KERNEL ); +if (name.data) { + name.len =HEXDIR_LEN ; + crp =nfs4_client_to_reclaim (name, nn); +if (crp) + crp->cr_clp = clp; +else +kfree (name.data); + }else { +dprintk ("%s: failed to allocate memory for name.data!\n" , + __func__); + } }vfs_fsync (nn->rec_file,0 ); }else {
@@ -353,6 +363,7 @@ nfsd4_remove_clid_dir(struct nfs4_client *clp)char dname[HEXDIR_LEN ];int status;struct nfsd_net *nn =net_generic (clp->net, nfsd_net_id); +struct xdr_netobj name;if (!nn->rec_file || !test_bit (NFSD4_CLIENT_STABLE , &clp->cl_flags))return ;
@@ -376,9 +387,17 @@ nfsd4_remove_clid_dir(struct nfs4_client *clp)vfs_fsync (nn->rec_file,0 );if (nn->in_grace) {/* remove reclaim record */ - crp =nfsd4_find_reclaim_client (dname, nn); -if (crp) -nfs4_remove_reclaim_record (crp, nn); + name.data =kmemdup (dname,HEXDIR_LEN ,GFP_KERNEL ); +if (name.data) { + name.len =HEXDIR_LEN ; + crp =nfsd4_find_reclaim_client (name, nn); +kfree (name.data); +if (crp) +nfs4_remove_reclaim_record (crp, nn); + }else { +dprintk ("%s: failed to allocate memory for name.data!\n" , + __func__); + } } } out_drop_write:
@@ -393,14 +412,31 @@ static int purge_old(struct dentry *parent, struct dentry *child, struct nfsd_net *nn) { int status; +struct xdr_netobj name; -if (nfs4_has_reclaimed_state (child->d_name.name, nn)) +if (child->d_name.len !=HEXDIR_LEN -1 ) { +printk ("%s: illegal name %pd in recovery directory\n" , + __func__, child); +/* Keep trying; maybe the others are OK: */ return 0 ; + } + name.data =kmemdup_nul (child->d_name.name, child->d_name.len,GFP_KERNEL ); +if (!name.data) { +dprintk ("%s: failed to allocate memory for name.data!\n" , + __func__); + goto out; + } + name.len =HEXDIR_LEN ; +if (nfs4_has_reclaimed_state (name, nn)) + goto out_free; status =vfs_rmdir (d_inode (parent), child);if (status)printk ("failed to remove client recovery directory %pd\n" , child); +out_free: +kfree (name.data); +out:/* Keep trying, success or failure: */ return 0 ; }
@@ -430,13 +466,24 @@ nfsd4_recdir_purge_old(struct nfsd_net *nn)static int load_recdir (struct dentry *parent,struct dentry *child,struct nfsd_net *nn) { +struct xdr_netobj name; +if (child->d_name.len !=HEXDIR_LEN -1 ) { -printk ("nfsd4: illegal name %pd in recovery directory\n" , - child); +printk ("%s: illegal name %pd in recovery directory\n" , + __func__, child);/* Keep trying; maybe the others are OK: */ return 0 ; } -nfs4_client_to_reclaim (child->d_name.name, nn); + name.data =kmemdup_nul (child->d_name.name, child->d_name.len,GFP_KERNEL ); +if (!name.data) { +dprintk ("%s: failed to allocate memory for name.data!\n" , + __func__); + goto out; + } + name.len =HEXDIR_LEN ; +if (!nfs4_client_to_reclaim (name, nn)) +kfree (name.data); +out:return 0 ; }
@@ -616,6 +663,7 @@ nfsd4_check_legacy_client(struct nfs4_client *clp)char dname[HEXDIR_LEN ];struct nfs4_client_reclaim *crp;struct nfsd_net *nn =net_generic (clp->net, nfsd_net_id); +struct xdr_netobj name;/* did we already find that this client is stable? */ if (test_bit (NFSD4_CLIENT_STABLE , &clp->cl_flags))
@@ -628,13 +676,22 @@ nfsd4_check_legacy_client(struct nfs4_client *clp) }/* look for it in the reclaim hashtable otherwise */ - crp =nfsd4_find_reclaim_client (dname, nn); + name.data =kmemdup (dname,HEXDIR_LEN ,GFP_KERNEL ); +if (!name.data) { +dprintk ("%s: failed to allocate memory for name.data!\n" , + __func__); + goto out_enoent; + } + name.len =HEXDIR_LEN ; + crp =nfsd4_find_reclaim_client (name, nn); +kfree (name.data);if (crp) {set_bit (NFSD4_CLIENT_STABLE , &clp->cl_flags); crp->cr_clp = clp;return 0 ; } +out_enoent:return -ENOENT ; }
fs/nfsd/nfs4state.c +15 -15
@@ -1058,9 +1058,9 @@ static unsigned int clientid_hashval(u32 id)return id &CLIENT_HASH_MASK ; } -static unsigned int clientstr_hashval (const char *name) +static unsigned int clientstr_hashval (struct xdr_netobj name) { -return opaque_hashval (name,8 ) &CLIENT_HASH_MASK ; +return opaque_hashval (name.data,8 ) &CLIENT_HASH_MASK ; }/*
@@ -2039,11 +2039,6 @@ compare_blob(const struct xdr_netobj *o1, const struct xdr_netobj *o2)return memcmp (o1->data, o2->data, o1->len); } -static int same_name (const char *n1,const char *n2) -{ -return 0 ==memcmp (n1, n2,HEXDIR_LEN ); -} -static int same_verf (nfs4_verifier *v1,nfs4_verifier *v2) {
@@ -6449,7 +6444,7 @@ alloc_reclaim(void) }bool -nfs4_has_reclaimed_state (const char *name,struct nfsd_net *nn) +nfs4_has_reclaimed_state (struct xdr_netobj name,struct nfsd_net *nn) {struct nfs4_client_reclaim *crp;
@@ -6459,20 +6454,24 @@ nfs4_has_reclaimed_state(const char *name, struct nfsd_net *nn)/* * failure => all reset bets are off, nfserr_no_grace... +* +* The caller is responsible for freeing name.data if NULL is returned (it +* will be freed in nfs4_remove_reclaim_record in the normal case). */ struct nfs4_client_reclaim * -nfs4_client_to_reclaim (const char *name,struct nfsd_net *nn) +nfs4_client_to_reclaim (struct xdr_netobj name,struct nfsd_net *nn) {unsigned int strhashval;struct nfs4_client_reclaim *crp; -dprintk ("NFSD nfs4_client_to_reclaim NAME: %.*s\n" ,HEXDIR_LEN , name); +dprintk ("NFSD nfs4_client_to_reclaim NAME: %.*s\n" , name.len, name.data); crp =alloc_reclaim ();if (crp) { strhashval =clientstr_hashval (name);INIT_LIST_HEAD (&crp->cr_strhash);list_add (&crp->cr_strhash, &nn->reclaim_str_hashtbl[strhashval]); -memcpy (crp->cr_recdir, name,HEXDIR_LEN ); + crp->cr_name.data = name.data; + crp->cr_name.len = name.len; crp->cr_clp =NULL ; nn->reclaim_str_hashtbl_size++; }
@@ -6483,6 +6482,7 @@ void nfs4_remove_reclaim_record(struct nfs4_client_reclaim *crp, struct nfsd_net *nn) { list_del(&crp->cr_strhash); +kfree (crp->cr_name.data);kfree (crp); nn->reclaim_str_hashtbl_size--; }
@@ -6506,16 +6506,16 @@ nfs4_release_reclaim(struct nfsd_net *nn)/* * called from OPEN, CLAIM_PREVIOUS with a new clientid. */ struct nfs4_client_reclaim * -nfsd4_find_reclaim_client (const char *recdir,struct nfsd_net *nn) +nfsd4_find_reclaim_client (struct xdr_netobj name,struct nfsd_net *nn) {unsigned int strhashval;struct nfs4_client_reclaim *crp =NULL ; -dprintk ("NFSD: nfs4_find_reclaim_client for recdir %s\n" , recdir); +dprintk ("NFSD: nfs4_find_reclaim_client for name %.*s\n" , name.len, name.data); - strhashval =clientstr_hashval (recdir); + strhashval =clientstr_hashval (name);list_for_each_entry (crp, &nn->reclaim_str_hashtbl[strhashval], cr_strhash) { -if (same_name (crp->cr_recdir, recdir)) { +if (compare_blob (&crp->cr_name, &name) ==0 ) {return crp; } }
fs/nfsd/state.h +4 -4
@@ -367,7 +367,7 @@ struct nfs4_client {struct nfs4_client_reclaim {struct list_head cr_strhash;/* hash by cr_name */ struct nfs4_client *cr_clp;/* pointer to associated clp */ -char cr_recdir[HEXDIR_LEN ];/* recover dir */ +struct xdr_netobj cr_name;/* recovery dir name */ };/* A reasonable value for REPLAY_ISIZE was estimated as follows:
@@ -619,7 +619,7 @@ void nfs4_put_stid(struct nfs4_stid *s);void nfs4_inc_and_copy_stateid (stateid_t *dst,struct nfs4_stid *stid);void nfs4_remove_reclaim_record (struct nfs4_client_reclaim *,struct nfsd_net *);extern void nfs4_release_reclaim (struct nfsd_net *); -extern struct nfs4_client_reclaim *nfsd4_find_reclaim_client (const char *recdir, +extern struct nfs4_client_reclaim *nfsd4_find_reclaim_client (struct xdr_netobj name,struct nfsd_net *nn);extern __be32 nfs4_check_open_reclaim (clientid_t *clid,struct nfsd4_compound_state *cstate,struct nfsd_net *nn);
@@ -634,9 +634,9 @@ extern void nfsd4_destroy_callback_queue(void);extern void nfsd4_shutdown_callback (struct nfs4_client *);extern void nfsd4_shutdown_copy (struct nfs4_client *clp);extern void nfsd4_prepare_cb_recall (struct nfs4_delegation *dp); -extern struct nfs4_client_reclaim *nfs4_client_to_reclaim (const char *name, +extern struct nfs4_client_reclaim *nfs4_client_to_reclaim (struct xdr_netobj name,struct nfsd_net *nn); -extern bool nfs4_has_reclaimed_state (const char *name,struct nfsd_net *nn); +extern bool nfs4_has_reclaimed_state (struct xdr_netobj name,struct nfsd_net *nn);struct nfs4_file *find_file (struct knfsd_fh *fh);void put_nfs4_file (struct nfs4_file *fi);
Signature · 1 line
2.17.1
Re: [PATCH RFC 2/4] nfsd: make nfs4_client_reclaim use an xdr_netobj instead of a fixed char array
Quoted 32 lines · Scott Mayhew
This will allow the
reclaim_str_hashtblto store either the recovery directory names used by the legacy client tracking code or the full client strings used by the nfsdcld client tracking code.Signed-off-by Scott Mayhew <[email protected]>--- fs/nfsd/nfs4recover.c | 79 +++++++++++++++++++++++++++++++++++++------ fs/nfsd/nfs4state.c | 30 ++++++++-------- fs/nfsd/state.h | 8 ++--- 3 files changed, 87 insertions(+), 30 deletions(-)fs/nfsd/nfs4recover.c +3
-3@@ -178,6 +178,7 @@ nfsd4_create_clid_dir(struct nfs4_client *clp)struct nfs4_client_reclaim *crp;int status;struct nfsd_net *nn =net_generic (clp->net, nfsd_net_id); +struct xdr_netobj name;if (test_and_set_bit (NFSD4_CLIENT_STABLE , &clp->cl_flags))return ;@@ -222,9 +223,18 @@ nfsd4_create_clid_dir(struct nfs4_client *clp)inode_unlock (d_inode (dir));if (status ==0 ) {if (nn->in_grace) { - crp =nfs4_client_to_reclaim (dname, nn); -if (crp) - crp->cr_clp = clp; + name.data =kmemdup (dname,HEXDIR_LEN ,GFP_KERNEL ); +if (name.data) {
fs/nfsd/nfs4recover.c nfsd4_create_clid_dir(struct nfs4_client *clp) · line 237
Quoted 10 lines
fs/nfsd/nfs4recover.c +10
-0+ name.len =HEXDIR_LEN ; + crp =nfs4_client_to_reclaim (name, nn); +if (crp) + crp->cr_clp = clp; +else +kfree (name.data); + }else { +dprintk ("%s: failed to allocate memory for name.data!\n" , + __func__); + }
The usual pattern for error handling is
try_something
if failed
goto error cleanup
try_something_else
if failed
goto error cleanup
...
and I find that a little easier to follow than this sort of:
try_something if succeeded try something_else if succeeded ...
Also, nfsd4_create_clid is getting a little long.
But, I don't know, I haven't tried, if the alternatives don't look like clear improvements then I can live with it.
The basic idea of the patch looks fine to me.
--b.
Quoted 255 lines
fs/nfsd/nfs4recover.c +55
-8}vfs_fsync (nn->rec_file,0 ); }else {@@ -353,6 +363,7 @@ nfsd4_remove_clid_dir(struct nfs4_client *clp)char dname[HEXDIR_LEN ];int status;struct nfsd_net *nn =net_generic (clp->net, nfsd_net_id); +struct xdr_netobj name;if (!nn->rec_file || !test_bit (NFSD4_CLIENT_STABLE , &clp->cl_flags))return ;@@ -376,9 +387,17 @@ nfsd4_remove_clid_dir(struct nfs4_client *clp)vfs_fsync (nn->rec_file,0 );if (nn->in_grace) {/* remove reclaim record */ - crp =nfsd4_find_reclaim_client (dname, nn); -if (crp) -nfs4_remove_reclaim_record (crp, nn); + name.data =kmemdup (dname,HEXDIR_LEN ,GFP_KERNEL ); +if (name.data) { + name.len =HEXDIR_LEN ; + crp =nfsd4_find_reclaim_client (name, nn); +kfree (name.data); +if (crp) +nfs4_remove_reclaim_record (crp, nn); + }else { +dprintk ("%s: failed to allocate memory for name.data!\n" , + __func__); + } } } out_drop_write:@@ -393,14 +412,31 @@ static int purge_old(struct dentry *parent, struct dentry *child, struct nfsd_net *nn) { int status; +struct xdr_netobj name; -if (nfs4_has_reclaimed_state (child->d_name.name, nn)) +if (child->d_name.len !=HEXDIR_LEN -1 ) { +printk ("%s: illegal name %pd in recovery directory\n" , + __func__, child); +/* Keep trying; maybe the others are OK: */ return 0 ; + } + name.data =kmemdup_nul (child->d_name.name, child->d_name.len,GFP_KERNEL ); +if (!name.data) { +dprintk ("%s: failed to allocate memory for name.data!\n" , + __func__); + goto out; + } + name.len =HEXDIR_LEN ; +if (nfs4_has_reclaimed_state (name, nn)) + goto out_free; status =vfs_rmdir (d_inode (parent), child);if (status)printk ("failed to remove client recovery directory %pd\n" , child); +out_free: +kfree (name.data); +out:/* Keep trying, success or failure: */ return 0 ; }@@ -430,13 +466,24 @@ nfsd4_recdir_purge_old(struct nfsd_net *nn)static int load_recdir (struct dentry *parent,struct dentry *child,struct nfsd_net *nn) { +struct xdr_netobj name; +if (child->d_name.len !=HEXDIR_LEN -1 ) { -printk ("nfsd4: illegal name %pd in recovery directory\n" , - child); +printk ("%s: illegal name %pd in recovery directory\n" , + __func__, child);/* Keep trying; maybe the others are OK: */ return 0 ; } -nfs4_client_to_reclaim (child->d_name.name, nn); + name.data =kmemdup_nul (child->d_name.name, child->d_name.len,GFP_KERNEL ); +if (!name.data) { +dprintk ("%s: failed to allocate memory for name.data!\n" , + __func__); + goto out; + } + name.len =HEXDIR_LEN ; +if (!nfs4_client_to_reclaim (name, nn)) +kfree (name.data); +out:return 0 ; }@@ -616,6 +663,7 @@ nfsd4_check_legacy_client(struct nfs4_client *clp)char dname[HEXDIR_LEN ];struct nfs4_client_reclaim *crp;struct nfsd_net *nn =net_generic (clp->net, nfsd_net_id); +struct xdr_netobj name;/* did we already find that this client is stable? */ if (test_bit (NFSD4_CLIENT_STABLE , &clp->cl_flags))@@ -628,13 +676,22 @@ nfsd4_check_legacy_client(struct nfs4_client *clp) }/* look for it in the reclaim hashtable otherwise */ - crp =nfsd4_find_reclaim_client (dname, nn); + name.data =kmemdup (dname,HEXDIR_LEN ,GFP_KERNEL ); +if (!name.data) { +dprintk ("%s: failed to allocate memory for name.data!\n" , + __func__); + goto out_enoent; + } + name.len =HEXDIR_LEN ; + crp =nfsd4_find_reclaim_client (name, nn); +kfree (name.data);if (crp) {set_bit (NFSD4_CLIENT_STABLE , &clp->cl_flags); crp->cr_clp = clp;return 0 ; } +out_enoent:return -ENOENT ; }fs/nfsd/nfs4state.c +15
-15@@ -1058,9 +1058,9 @@ static unsigned int clientid_hashval(u32 id)return id &CLIENT_HASH_MASK ; } -static unsigned int clientstr_hashval (const char *name) +static unsigned int clientstr_hashval (struct xdr_netobj name) { -return opaque_hashval (name,8 ) &CLIENT_HASH_MASK ; +return opaque_hashval (name.data,8 ) &CLIENT_HASH_MASK ; }/* @@ -2039,11 +2039,6 @@ compare_blob(const struct xdr_netobj *o1, const struct xdr_netobj *o2)return memcmp (o1->data, o2->data, o1->len); } -static int same_name (const char *n1,const char *n2) -{ -return 0 ==memcmp (n1, n2,HEXDIR_LEN ); -} -static int same_verf (nfs4_verifier *v1,nfs4_verifier *v2) {@@ -6449,7 +6444,7 @@ alloc_reclaim(void) }bool -nfs4_has_reclaimed_state (const char *name,struct nfsd_net *nn) +nfs4_has_reclaimed_state (struct xdr_netobj name,struct nfsd_net *nn) {struct nfs4_client_reclaim *crp;@@ -6459,20 +6454,24 @@ nfs4_has_reclaimed_state(const char *name, struct nfsd_net *nn)/* * failure => all reset bets are off, nfserr_no_grace... +* +* The caller is responsible for freeing name.data if NULL is returned (it +* will be freed in nfs4_remove_reclaim_record in the normal case). */ struct nfs4_client_reclaim * -nfs4_client_to_reclaim (const char *name,struct nfsd_net *nn) +nfs4_client_to_reclaim (struct xdr_netobj name,struct nfsd_net *nn) {unsigned int strhashval;struct nfs4_client_reclaim *crp; -dprintk ("NFSD nfs4_client_to_reclaim NAME: %.*s\n" ,HEXDIR_LEN , name); +dprintk ("NFSD nfs4_client_to_reclaim NAME: %.*s\n" , name.len, name.data); crp =alloc_reclaim ();if (crp) { strhashval =clientstr_hashval (name);INIT_LIST_HEAD (&crp->cr_strhash);list_add (&crp->cr_strhash, &nn->reclaim_str_hashtbl[strhashval]); -memcpy (crp->cr_recdir, name,HEXDIR_LEN ); + crp->cr_name.data = name.data; + crp->cr_name.len = name.len; crp->cr_clp =NULL ; nn->reclaim_str_hashtbl_size++; }@@ -6483,6 +6482,7 @@ void nfs4_remove_reclaim_record(struct nfs4_client_reclaim *crp, struct nfsd_net *nn) { list_del(&crp->cr_strhash); +kfree (crp->cr_name.data);kfree (crp); nn->reclaim_str_hashtbl_size--; }@@ -6506,16 +6506,16 @@ nfs4_release_reclaim(struct nfsd_net *nn)/* * called from OPEN, CLAIM_PREVIOUS with a new clientid. */ struct nfs4_client_reclaim * -nfsd4_find_reclaim_client (const char *recdir,struct nfsd_net *nn) +nfsd4_find_reclaim_client (struct xdr_netobj name,struct nfsd_net *nn) {unsigned int strhashval;struct nfs4_client_reclaim *crp =NULL ; -dprintk ("NFSD: nfs4_find_reclaim_client for recdir %s\n" , recdir); +dprintk ("NFSD: nfs4_find_reclaim_client for name %.*s\n" , name.len, name.data); - strhashval =clientstr_hashval (recdir); + strhashval =clientstr_hashval (name);list_for_each_entry (crp, &nn->reclaim_str_hashtbl[strhashval], cr_strhash) { -if (same_name (crp->cr_recdir, recdir)) { +if (compare_blob (&crp->cr_name, &name) ==0 ) {return crp; } }fs/nfsd/state.h +4
-4@@ -367,7 +367,7 @@ struct nfs4_client {struct nfs4_client_reclaim {struct list_head cr_strhash;/* hash by cr_name */ struct nfs4_client *cr_clp;/* pointer to associated clp */ -char cr_recdir[HEXDIR_LEN ];/* recover dir */ +struct xdr_netobj cr_name;/* recovery dir name */ };/* A reasonable value for REPLAY_ISIZE was estimated as follows: @@ -619,7 +619,7 @@ void nfs4_put_stid(struct nfs4_stid *s);void nfs4_inc_and_copy_stateid (stateid_t *dst,struct nfs4_stid *stid);void nfs4_remove_reclaim_record (struct nfs4_client_reclaim *,struct nfsd_net *);extern void nfs4_release_reclaim (struct nfsd_net *); -extern struct nfs4_client_reclaim *nfsd4_find_reclaim_client (const char *recdir, +extern struct nfs4_client_reclaim *nfsd4_find_reclaim_client (struct xdr_netobj name,struct nfsd_net *nn);extern __be32 nfs4_check_open_reclaim (clientid_t *clid,struct nfsd4_compound_state *cstate,struct nfsd_net *nn);@@ -634,9 +634,9 @@ extern void nfsd4_destroy_callback_queue(void);extern void nfsd4_shutdown_callback (struct nfs4_client *);extern void nfsd4_shutdown_copy (struct nfs4_client *clp);extern void nfsd4_prepare_cb_recall (struct nfs4_delegation *dp); -extern struct nfs4_client_reclaim *nfs4_client_to_reclaim (const char *name, +extern struct nfs4_client_reclaim *nfs4_client_to_reclaim (struct xdr_netobj name,struct nfsd_net *nn); -extern bool nfs4_has_reclaimed_state (const char *name,struct nfsd_net *nn); +extern bool nfs4_has_reclaimed_state (struct xdr_netobj name,struct nfsd_net *nn);struct nfs4_file *find_file (struct knfsd_fh *fh);void put_nfs4_file (struct nfs4_file *fi);Signature · 1 line
2.17.1
RFC PATCH 3/4 nfsd: un-deprecate nfsdcld
In reply to Scott MayhewWhen nfsdcld was released, it was quickly deprecated in favor of the nfsdcltrack usermodehelper, so as to not require another running daemon. That prevents NFSv4 clients from reclaiming locks from nfsd's running in containers, since neither nfsdcltrack nor the legacy client tracking code work in containers.
This commit un-deprecates the use of nfsdcld, with one twist: we will populate the reclaim_str_hashtbl on startup.
During client tracking initialization, do an upcall ("GraceStart") to nfsdcld to get a list of clients from the database. nfsdcld will do one downcall with a status of -EINPROGRESS for each client record in the database, which in turn will cause an nfs4_client_reclaim to be added to the reclaim_str_hashtbl. When complete, nfsdcld will do a final downcall with a status of 0.
This will save nfsd from having to do an upcall to the daemon during nfs4_check_open_reclaim() processing.
Even though nfsdcld was quickly deprecated, there is a very small chance of old nfsdcld daemons running in the wild. These will respond to the new "GraceStart" upcall with -EOPNOTSUPP, in which case we will log a message and fall back to the original nfsdcld tracking ops (now called nfsd4_cld_tracking_ops_v0).
--- fs/nfsd/nfs4recover.c | 225 ++++++++++++++++++++++++++++++++-- include/uapi/linux/nfsd/cld.h | 1 + 2 files changed, 218 insertions(+), 8 deletions(-)
fs/nfsd/nfs4recover.c +217 -8
@@ -763,6 +763,34 @@ cld_pipe_upcall(struct rpc_pipe *pipe, struct cld_msg *cmsg)return ret; } +static ssize_t +__cld_pipe_inprogress_downcall (const struct cld_msg __user *cmsg, +struct nfsd_net *nn) +{ +uint8_t cmd; +struct xdr_netobj name; +uint16_t namelen; + +if (get_user (cmd, &cmsg->cm_cmd)) { +dprintk ("%s: error when copying cmd from userspace" , __func__); +return -EFAULT ; + } +if (cmd == Cld_GraceStart) { +if (get_user (namelen, &cmsg->cm_u.cm_name.cn_len)) +return -EFAULT ; + name.data =memdup_user (&cmsg->cm_u.cm_name.cn_id, namelen); +if (IS_ERR_OR_NULL (name.data)) +return -EFAULT ; + name.len = namelen; +if (!nfs4_client_to_reclaim (name, nn)) { +kfree (name.data); +return -EFAULT ; + } +return sizeof (*cmsg); + } +return -EFAULT ; +} +static ssize_t cld_pipe_downcall (struct file *filp,const char __user *src,size_t mlen) {
@@ -772,6 +800,7 @@ cld_pipe_downcall(struct file *filp, const char __user *src, size_t mlen)struct nfsd_net *nn =net_generic (file_inode (filp)->i_sb->s_fs_info, nfsd_net_id);struct cld_net *cn = nn->cld_net; +int16_t status;if (mlen !=sizeof (*cmsg)) { dprintk("%s: got %zu bytes, expected %zu\n", __func__, mlen,
@@ -785,13 +814,24 @@ cld_pipe_downcall(struct file *filp, const char __user *src, size_t mlen)return -EFAULT ; } +/* +* copy the status so we know whether to remove the upcall from the +* list (for -EINPROGRESS, we just want to make sure the xid is +* valid, not remove the upcall from the list) +*/ +if (get_user (status, &cmsg->cm_status)) { +dprintk ("%s: error when copying status from userspace" , __func__); +return -EFAULT ; + } +/* walk the list and find corresponding xid */ cup =NULL ;spin_lock (&cn->cn_lock);list_for_each_entry (tmp, &cn->cn_list, cu_list) {if (get_unaligned (&tmp->cu_msg.cm_xid) == xid) { cup = tmp; -list_del_init (&cup->cu_list); +if (status != -EINPROGRESS ) +list_del_init (&cup->cu_list);break ; } }
@@ -803,6 +843,9 @@ cld_pipe_downcall(struct file *filp, const char __user *src, size_t mlen)return -EINVAL ; } +if (status == -EINPROGRESS ) +return __cld_pipe_inprogress_downcall (cmsg, nn); +if (copy_from_user (&cup->cu_msg, src, mlen) !=0 )return -EFAULT ;
@@ -1049,9 +1092,14 @@ nfsd4_cld_remove(struct nfs4_client *clp) "record from stable storage: %d\n", ret); } -/* Check for presence of a record, and update its timestamp */ +/* +* For older nfsdcld's that do not allow us to "slurp" the clients +* from the tracking database during startup. +* +* Check for presence of a record, and update its timestamp +*/ static int -nfsd4_cld_check (struct nfs4_client *clp) +nfsd4_cld_check_v0 (struct nfs4_client *clp) {int ret;struct cld_upcall *cup;
@@ -1084,8 +1132,61 @@ nfsd4_cld_check(struct nfs4_client *clp)return ret; } +/* +* For newer nfsdcld's that allow us to "slurp" the clients +* from the tracking database during startup. +* +* Check for presence of a record in the reclaim_str_hashtbl +*/ +static int +nfsd4_cld_check (struct nfs4_client *clp) +{ +struct nfs4_client_reclaim *crp; +struct nfsd_net *nn =net_generic (clp->net, nfsd_net_id); + +/* did we already find that this client is stable? */ +if (test_bit (NFSD4_CLIENT_STABLE , &clp->cl_flags)) +return 0 ; + +/* look for it in the reclaim hashtable otherwise */ + crp =nfsd4_find_reclaim_client (clp->cl_name, nn); +if (crp) { + crp->cr_clp = clp; +return 0 ; + } + +return -ENOENT ; +} + +static int +nfsd4_cld_grace_start (struct nfsd_net *nn) +{ +int ret; +struct cld_upcall *cup; +struct cld_net *cn = nn->cld_net; + + cup =alloc_cld_upcall (cn); +if (!cup) { + ret = -ENOMEM ; + goto out_err; + } + + cup->cu_msg.cm_cmd = Cld_GraceStart; + ret =cld_pipe_upcall (cn->cn_pipe, &cup->cu_msg); +if (!ret) + ret = cup->cu_msg.cm_status; + +free_cld_upcall (cup); +out_err: +if (ret) +dprintk ("%s: Unable to get clients from userspace: %d\n" , + __func__, ret); +return ret; +} + +/* For older nfsdcld's that need cm_gracetime */ static void -nfsd4_cld_grace_done (struct nfsd_net *nn) +nfsd4_cld_grace_done_v0 (struct nfsd_net *nn) {int ret;struct cld_upcall *cup;
@@ -1109,11 +1210,118 @@ nfsd4_cld_grace_done(struct nfsd_net *nn)printk (KERN_ERR "NFSD: Unable to end grace period: %d\n" , ret); } -static const struct nfsd4_client_tracking_ops nfsd4_cld_tracking_ops = { +/* +* For newer nfsdcld's that do not need cm_gracetime. We also need to call +* nfs4_release_reclaim() to clear out the reclaim_str_hashtbl. +*/ +static void +nfsd4_cld_grace_done (struct nfsd_net *nn) +{ +int ret; +struct cld_upcall *cup; +struct cld_net *cn = nn->cld_net; + + cup =alloc_cld_upcall (cn); +if (!cup) { + ret = -ENOMEM ; + goto out_err; + } + + cup->cu_msg.cm_cmd = Cld_GraceDone; + ret =cld_pipe_upcall (cn->cn_pipe, &cup->cu_msg); +if (!ret) + ret = cup->cu_msg.cm_status; + +free_cld_upcall (cup); +out_err: +nfs4_release_reclaim (nn); +if (ret) +printk (KERN_ERR "NFSD: Unable to end grace period: %d\n" , ret); +} + +static int +nfs4_cld_state_init (struct net *net) +{ +struct nfsd_net *nn =net_generic (net, nfsd_net_id); +int i; + + nn->reclaim_str_hashtbl =kmalloc_array (CLIENT_HASH_SIZE , +sizeof (struct list_head ), +GFP_KERNEL ); +if (!nn->reclaim_str_hashtbl) +return -ENOMEM ; + +for (i =0 ; i <CLIENT_HASH_SIZE ; i++) +INIT_LIST_HEAD (&nn->reclaim_str_hashtbl[i]); + nn->reclaim_str_hashtbl_size =0 ; + +return 0 ; +} + +static void +nfs4_cld_state_shutdown (struct net *net) +{ +struct nfsd_net *nn =net_generic (net, nfsd_net_id); + +kfree (nn->reclaim_str_hashtbl); +} + +static int +nfsd4_cld_tracking_init (struct net *net) +{ +int status; +struct nfsd_net *nn =net_generic (net, nfsd_net_id); + + status =nfs4_cld_state_init (net); +if (status) +return status; + + status =nfsd4_init_cld_pipe (net); +if (status) + goto err_shutdown; + + status =nfsd4_cld_grace_start (nn); +if (status) { +if (status == -EOPNOTSUPP ) +printk (KERN_WARNING "NFSD: Please upgrade nfsdcld.\n" ); +nfs4_release_reclaim (nn); + goto err_remove; + } +return 0 ; + +err_remove: +nfsd4_remove_cld_pipe (net); +err_shutdown: +nfs4_cld_state_shutdown (net); +return status; +} + +static void +nfsd4_cld_tracking_exit (struct net *net) +{ +struct nfsd_net *nn =net_generic (net, nfsd_net_id); + +nfs4_release_reclaim (nn); +nfsd4_remove_cld_pipe (net); +nfs4_cld_state_shutdown (net); +} + +/* For older nfsdcld's */ +static const struct nfsd4_client_tracking_ops nfsd4_cld_tracking_ops_v0 = { .init = nfsd4_init_cld_pipe, .exit = nfsd4_remove_cld_pipe, .create = nfsd4_cld_create, .remove = nfsd4_cld_remove, + .check = nfsd4_cld_check_v0, + .grace_done = nfsd4_cld_grace_done_v0, +}; + +/* For newer nfsdcld's */ +static const struct nfsd4_client_tracking_ops nfsd4_cld_tracking_ops = { + .init = nfsd4_cld_tracking_init, + .exit = nfsd4_cld_tracking_exit, + .create = nfsd4_cld_create, + .remove = nfsd4_cld_remove, .check = nfsd4_cld_check, .grace_done = nfsd4_cld_grace_done, };
@@ -1498,9 +1706,10 @@ nfsd4_client_tracking_init(struct net *net)/* Finally, try to use nfsdcld */ nn->client_tracking_ops = &nfsd4_cld_tracking_ops; -printk (KERN_WARNING "NFSD: the nfsdcld client tracking upcall will be " -"removed in 3.10. Please transition to using " -"nfsdcltrack.\n" ); + status = nn->client_tracking_ops->init (net); +if (!status) +return status; + nn->client_tracking_ops = &nfsd4_cld_tracking_ops_v0; do_init: status = nn->client_tracking_ops->init (net);if (status) {
include/uapi/linux/nfsd/cld.h +1 -0
@@ -36,6 +36,7 @@ enum cld_command { Cld_Remove,/* remove record of this cm_id */ Cld_Check,/* is this cm_id allowed? */ Cld_GraceDone,/* grace period is complete */ + Cld_GraceStart, };/* representation of long-form NFSv4 client ID */
Signature · 1 line
2.17.1
Re: [PATCH RFC 3/4] nfsd: un-deprecate nfsdcld
Quoted 9 lines · Scott Mayhew
When nfsdcld was released, it was quickly deprecated in favor of the nfsdcltrack usermodehelper, so as to not require another running daemon. That prevents NFSv4 clients from reclaiming locks from nfsd's running in containers, since neither nfsdcltrack nor the legacy client tracking code work in containers.
This commit un-deprecates the use of nfsdcld, with one twist: we will populate the
reclaim_str_hashtblon startup.
Quoted 6 lines
During client tracking initialization, do an upcall ("GraceStart") to nfsdcld to get a list of clients from the database. nfsdcld will do one downcall with a status of
-EINPROGRESSfor each client record in the database, which in turn will cause annfs4_client_reclaimto be added to thereclaim_str_hashtbl. When complete, nfsdcld will do a final downcall with a status of 0.
Kind of a funny convention, but OK, sounds like it should work.
Quoted 8 lines
This will save nfsd from having to do an upcall to the daemon during
nfs4_check_open_reclaim()processing.Even though nfsdcld was quickly deprecated, there is a very small chance of old nfsdcld daemons running in the wild. These will respond to the new "GraceStart" upcall with
-EOPNOTSUPP, in which case we will log a message and fall back to the original nfsdcld tracking ops (now callednfsd4_cld_tracking_ops_v0).
Sounds good.
The code looks OK to me.
--b.
Quoted 318 lines
Signed-off-by Scott Mayhew <[email protected]>--- fs/nfsd/nfs4recover.c | 225 ++++++++++++++++++++++++++++++++-- include/uapi/linux/nfsd/cld.h | 1 + 2 files changed, 218 insertions(+), 8 deletions(-)fs/nfsd/nfs4recover.c +217
-8@@ -763,6 +763,34 @@ cld_pipe_upcall(struct rpc_pipe *pipe, struct cld_msg *cmsg)return ret; } +static ssize_t +__cld_pipe_inprogress_downcall (const struct cld_msg __user *cmsg, +struct nfsd_net *nn) +{ +uint8_t cmd; +struct xdr_netobj name; +uint16_t namelen; + +if (get_user (cmd, &cmsg->cm_cmd)) { +dprintk ("%s: error when copying cmd from userspace" , __func__); +return -EFAULT ; + } +if (cmd == Cld_GraceStart) { +if (get_user (namelen, &cmsg->cm_u.cm_name.cn_len)) +return -EFAULT ; + name.data =memdup_user (&cmsg->cm_u.cm_name.cn_id, namelen); +if (IS_ERR_OR_NULL (name.data)) +return -EFAULT ; + name.len = namelen; +if (!nfs4_client_to_reclaim (name, nn)) { +kfree (name.data); +return -EFAULT ; + } +return sizeof (*cmsg); + } +return -EFAULT ; +} +static ssize_t cld_pipe_downcall (struct file *filp,const char __user *src,size_t mlen) {@@ -772,6 +800,7 @@ cld_pipe_downcall(struct file *filp, const char __user *src, size_t mlen)struct nfsd_net *nn =net_generic (file_inode (filp)->i_sb->s_fs_info, nfsd_net_id);struct cld_net *cn = nn->cld_net; +int16_t status;if (mlen !=sizeof (*cmsg)) { dprintk("%s: got %zu bytes, expected %zu\n", __func__, mlen,@@ -785,13 +814,24 @@ cld_pipe_downcall(struct file *filp, const char __user *src, size_t mlen)return -EFAULT ; } +/* +* copy the status so we know whether to remove the upcall from the +* list (for -EINPROGRESS, we just want to make sure the xid is +* valid, not remove the upcall from the list) +*/ +if (get_user (status, &cmsg->cm_status)) { +dprintk ("%s: error when copying status from userspace" , __func__); +return -EFAULT ; + } +/* walk the list and find corresponding xid */ cup =NULL ;spin_lock (&cn->cn_lock);list_for_each_entry (tmp, &cn->cn_list, cu_list) {if (get_unaligned (&tmp->cu_msg.cm_xid) == xid) { cup = tmp; -list_del_init (&cup->cu_list); +if (status != -EINPROGRESS ) +list_del_init (&cup->cu_list);break ; } }@@ -803,6 +843,9 @@ cld_pipe_downcall(struct file *filp, const char __user *src, size_t mlen)return -EINVAL ; } +if (status == -EINPROGRESS ) +return __cld_pipe_inprogress_downcall (cmsg, nn); +if (copy_from_user (&cup->cu_msg, src, mlen) !=0 )return -EFAULT ;@@ -1049,9 +1092,14 @@ nfsd4_cld_remove(struct nfs4_client *clp) "record from stable storage: %d\n", ret); } -/* Check for presence of a record, and update its timestamp */ +/* +* For older nfsdcld's that do not allow us to "slurp" the clients +* from the tracking database during startup. +* +* Check for presence of a record, and update its timestamp +*/ static int -nfsd4_cld_check (struct nfs4_client *clp) +nfsd4_cld_check_v0 (struct nfs4_client *clp) {int ret;struct cld_upcall *cup;@@ -1084,8 +1132,61 @@ nfsd4_cld_check(struct nfs4_client *clp)return ret; } +/* +* For newer nfsdcld's that allow us to "slurp" the clients +* from the tracking database during startup. +* +* Check for presence of a record in the reclaim_str_hashtbl +*/ +static int +nfsd4_cld_check (struct nfs4_client *clp) +{ +struct nfs4_client_reclaim *crp; +struct nfsd_net *nn =net_generic (clp->net, nfsd_net_id); + +/* did we already find that this client is stable? */ +if (test_bit (NFSD4_CLIENT_STABLE , &clp->cl_flags)) +return 0 ; + +/* look for it in the reclaim hashtable otherwise */ + crp =nfsd4_find_reclaim_client (clp->cl_name, nn); +if (crp) { + crp->cr_clp = clp; +return 0 ; + } + +return -ENOENT ; +} + +static int +nfsd4_cld_grace_start (struct nfsd_net *nn) +{ +int ret; +struct cld_upcall *cup; +struct cld_net *cn = nn->cld_net; + + cup =alloc_cld_upcall (cn); +if (!cup) { + ret = -ENOMEM ; + goto out_err; + } + + cup->cu_msg.cm_cmd = Cld_GraceStart; + ret =cld_pipe_upcall (cn->cn_pipe, &cup->cu_msg); +if (!ret) + ret = cup->cu_msg.cm_status; + +free_cld_upcall (cup); +out_err: +if (ret) +dprintk ("%s: Unable to get clients from userspace: %d\n" , + __func__, ret); +return ret; +} + +/* For older nfsdcld's that need cm_gracetime */ static void -nfsd4_cld_grace_done (struct nfsd_net *nn) +nfsd4_cld_grace_done_v0 (struct nfsd_net *nn) {int ret;struct cld_upcall *cup;@@ -1109,11 +1210,118 @@ nfsd4_cld_grace_done(struct nfsd_net *nn)printk (KERN_ERR "NFSD: Unable to end grace period: %d\n" , ret); } -static const struct nfsd4_client_tracking_ops nfsd4_cld_tracking_ops = { +/* +* For newer nfsdcld's that do not need cm_gracetime. We also need to call +* nfs4_release_reclaim() to clear out the reclaim_str_hashtbl. +*/ +static void +nfsd4_cld_grace_done (struct nfsd_net *nn) +{ +int ret; +struct cld_upcall *cup; +struct cld_net *cn = nn->cld_net; + + cup =alloc_cld_upcall (cn); +if (!cup) { + ret = -ENOMEM ; + goto out_err; + } + + cup->cu_msg.cm_cmd = Cld_GraceDone; + ret =cld_pipe_upcall (cn->cn_pipe, &cup->cu_msg); +if (!ret) + ret = cup->cu_msg.cm_status; + +free_cld_upcall (cup); +out_err: +nfs4_release_reclaim (nn); +if (ret) +printk (KERN_ERR "NFSD: Unable to end grace period: %d\n" , ret); +} + +static int +nfs4_cld_state_init (struct net *net) +{ +struct nfsd_net *nn =net_generic (net, nfsd_net_id); +int i; + + nn->reclaim_str_hashtbl =kmalloc_array (CLIENT_HASH_SIZE , +sizeof (struct list_head ), +GFP_KERNEL ); +if (!nn->reclaim_str_hashtbl) +return -ENOMEM ; + +for (i =0 ; i <CLIENT_HASH_SIZE ; i++) +INIT_LIST_HEAD (&nn->reclaim_str_hashtbl[i]); + nn->reclaim_str_hashtbl_size =0 ; + +return 0 ; +} + +static void +nfs4_cld_state_shutdown (struct net *net) +{ +struct nfsd_net *nn =net_generic (net, nfsd_net_id); + +kfree (nn->reclaim_str_hashtbl); +} + +static int +nfsd4_cld_tracking_init (struct net *net) +{ +int status; +struct nfsd_net *nn =net_generic (net, nfsd_net_id); + + status =nfs4_cld_state_init (net); +if (status) +return status; + + status =nfsd4_init_cld_pipe (net); +if (status) + goto err_shutdown; + + status =nfsd4_cld_grace_start (nn); +if (status) { +if (status == -EOPNOTSUPP ) +printk (KERN_WARNING "NFSD: Please upgrade nfsdcld.\n" ); +nfs4_release_reclaim (nn); + goto err_remove; + } +return 0 ; + +err_remove: +nfsd4_remove_cld_pipe (net); +err_shutdown: +nfs4_cld_state_shutdown (net); +return status; +} + +static void +nfsd4_cld_tracking_exit (struct net *net) +{ +struct nfsd_net *nn =net_generic (net, nfsd_net_id); + +nfs4_release_reclaim (nn); +nfsd4_remove_cld_pipe (net); +nfs4_cld_state_shutdown (net); +} + +/* For older nfsdcld's */ +static const struct nfsd4_client_tracking_ops nfsd4_cld_tracking_ops_v0 = { .init = nfsd4_init_cld_pipe, .exit = nfsd4_remove_cld_pipe, .create = nfsd4_cld_create, .remove = nfsd4_cld_remove, + .check = nfsd4_cld_check_v0, + .grace_done = nfsd4_cld_grace_done_v0, +}; + +/* For newer nfsdcld's */ +static const struct nfsd4_client_tracking_ops nfsd4_cld_tracking_ops = { + .init = nfsd4_cld_tracking_init, + .exit = nfsd4_cld_tracking_exit, + .create = nfsd4_cld_create, + .remove = nfsd4_cld_remove, .check = nfsd4_cld_check, .grace_done = nfsd4_cld_grace_done, };@@ -1498,9 +1706,10 @@ nfsd4_client_tracking_init(struct net *net)/* Finally, try to use nfsdcld */ nn->client_tracking_ops = &nfsd4_cld_tracking_ops; -printk (KERN_WARNING "NFSD: the nfsdcld client tracking upcall will be " -"removed in 3.10. Please transition to using " -"nfsdcltrack.\n" ); + status = nn->client_tracking_ops->init (net); +if (!status) +return status; + nn->client_tracking_ops = &nfsd4_cld_tracking_ops_v0; do_init: status = nn->client_tracking_ops->init (net);if (status) {include/uapi/linux/nfsd/cld.h +1
-0@@ -36,6 +36,7 @@ enum cld_command { Cld_Remove,/* remove record of this cm_id */ Cld_Check,/* is this cm_id allowed? */ Cld_GraceDone,/* grace period is complete */ + Cld_GraceStart, };/* representation of long-form NFSv4 client ID */ Signature · 1 line
2.17.1
RFC PATCH 4/4 nfsd: keep a tally of RECLAIM_COMPLETE operations when using nfsdcld
In reply to Scott MayhewWhen using nfsdcld for NFSv4 client tracking, track the number of RECLAIM_COMPLETE operations we receive from "known" clients to help in deciding if we can lift the grace period early (or whether we need to start a v4 grace period at all).
--- fs/nfsd/netns.h | 3 +++ fs/nfsd/nfs4recover.c | 5 +++++ fs/nfsd/nfs4state.c | 52 +++++++++++++++++++++++++++++++++++++++++++ fs/nfsd/nfsctl.c | 1 + 4 files changed, 61 insertions(+)
fs/nfsd/netns.h +3 -0
@@ -104,6 +104,9 @@ struct nfsd_net {time_t nfsd4_grace;bool somebody_reclaimed; +bool track_reclaim_completes; +atomic_t nr_reclaim_complete; +bool nfsd_net_up;bool lockd_up;
fs/nfsd/nfs4recover.c +5 -0
@@ -1235,6 +1235,7 @@ nfsd4_cld_grace_done(struct nfsd_net *nn)free_cld_upcall (cup); out_err:nfs4_release_reclaim (nn); +atomic_set (&nn->nr_reclaim_complete,0 );if (ret)printk (KERN_ERR "NFSD: Unable to end grace period: %d\n" , ret); }
@@ -1254,6 +1255,8 @@ nfs4_cld_state_init(struct net *net)for (i =0 ; i <CLIENT_HASH_SIZE ; i++)INIT_LIST_HEAD (&nn->reclaim_str_hashtbl[i]); nn->reclaim_str_hashtbl_size =0 ; + nn->track_reclaim_completes = true; +atomic_set (&nn->nr_reclaim_complete,0 );return 0 ; }
@@ -1263,6 +1266,7 @@ nfs4_cld_state_shutdown(struct net *net) {struct nfsd_net *nn =net_generic (net, nfsd_net_id); + nn->track_reclaim_completes = false;kfree (nn->reclaim_str_hashtbl); }
@@ -1302,6 +1306,7 @@ nfsd4_cld_tracking_exit(struct net *net)struct nfsd_net *nn =net_generic (net, nfsd_net_id);nfs4_release_reclaim (nn); +atomic_set (&nn->nr_reclaim_complete,0 );nfsd4_remove_cld_pipe (net);nfs4_cld_state_shutdown (net); }
fs/nfsd/nfs4state.c +52 -0
90 lines not on this page · raw message
fs/nfsd/nfsctl.c +1 -0
@@ -1240,6 +1240,7 @@ static __net_init int nfsd_init_net(struct net *net) nn->nfsd4_lease =45 ;/* default lease time */ nn->nfsd4_grace =45 ; nn->somebody_reclaimed = false; + nn->track_reclaim_completes = false; nn->clverifier_counter =prandom_u32 (); nn->clientid_counter =prandom_u32 (); nn->s2s_cp_cl_id = nn->clientid_counter++;
Signature · 1 line
2.17.1
Thanks for doing this, we've really been wanting it. I should be able to get to it in the next few days....
Out of curiosity, what are you using for testing?
--b.
Quoted 40 lines · Scott Mayhew
When nfsdcld was released, it was quickly deprecated in favor of the nfsdcltrack usermodehelper, so as to not require another running daemon. That prevents NFSv4 clients from reclaiming locks from nfsd's running in containers, since neither nfsdcltrack nor the legacy client tracking code work in containers. These patches un-deprecate the use of nfsdcld for NFSv4 client tracking.
These patches are intended to go alongside some nfs-utils patches that introduce an enhancement that allows nfsd to "slurp" up the client records during client tracking initialization and store them internally in hash table. This enables nfsd to check whether an NFSv4 client is allowed to reclaim without having to do an upcall to nfsdcld. It also allows nfsd to decide to end the v4 grace period early if the number of
RECLAIM_COMPLETEoperations it has received from "known" clients is equal to the number of entries in the hash table. It also allows nfsd to skip the v4 grace period altogether if it knows there are no clients allowed to reclaim.There is a fallback to allow nfsd to continue to work with older nfsdcld daemons in the event that any are out in the wild (unlikely). Everything should work fine except nfsd will not be able to exit the grace period early or skip the grace period altogether.
Scott Mayhew (4):
nfsd: fix a warning in__cld_pipe_upcall()
nfsd: makenfs4_client_reclaimuse anxdr_netobjinstead of a fixed
char array
nfsd: un-deprecate nfsdcld
nfsd: keep a tally ofRECLAIM_COMPLETEoperations when using nfsdcldfs/nfsd/netns.h | 3 + fs/nfsd/nfs4recover.c | 326 ++++++++++++++++++++++++++++++---- fs/nfsd/nfs4state.c | 82 +++++++-- fs/nfsd/nfsctl.c | 1 + fs/nfsd/state.h | 8 +- include/uapi/linux/nfsd/cld.h | 1 + 6 files changed, 372 insertions(+), 49 deletions(-)Signature · 1 line
2.17.1
Quoted 4 lines · J. Bruce Fields
Thanks for doing this, we've really been wanting it. I should be able to get to it in the next few days....
Out of curiosity, what are you using for testing?
For initially testing the GraceStart upcall, I'd just loaded a bunch of bogus clientid's into the database and wrote a crash macro to dump out the reclaim_str_hashtbl to make sure everything got there on startup.
For testing nfsd in a container, right now I'm using plain docker. I have one container running nfsd. I made another image that just runs cthon against an ip that you pass in via an environment variable. I start up a bunch of containers using the cthon image and then periodically restart the nfsd container. I'd like to try this stuff out on OpenShift at some point too.
I'm open to other ideas for testing if you have any.
-Scott
Quoted 44 lines
--b.
Quoted 40 lines · Scott Mayhew
When nfsdcld was released, it was quickly deprecated in favor of the nfsdcltrack usermodehelper, so as to not require another running daemon. That prevents NFSv4 clients from reclaiming locks from nfsd's running in containers, since neither nfsdcltrack nor the legacy client tracking code work in containers. These patches un-deprecate the use of nfsdcld for NFSv4 client tracking.
These patches are intended to go alongside some nfs-utils patches that introduce an enhancement that allows nfsd to "slurp" up the client records during client tracking initialization and store them internally in hash table. This enables nfsd to check whether an NFSv4 client is allowed to reclaim without having to do an upcall to nfsdcld. It also allows nfsd to decide to end the v4 grace period early if the number of
RECLAIM_COMPLETEoperations it has received from "known" clients is equal to the number of entries in the hash table. It also allows nfsd to skip the v4 grace period altogether if it knows there are no clients allowed to reclaim.There is a fallback to allow nfsd to continue to work with older nfsdcld daemons in the event that any are out in the wild (unlikely). Everything should work fine except nfsd will not be able to exit the grace period early or skip the grace period altogether.
Scott Mayhew (4):
nfsd: fix a warning in__cld_pipe_upcall()
nfsd: makenfs4_client_reclaimuse anxdr_netobjinstead of a fixed
char array
nfsd: un-deprecate nfsdcld
nfsd: keep a tally ofRECLAIM_COMPLETEoperations when using nfsdcldfs/nfsd/netns.h | 3 + fs/nfsd/nfs4recover.c | 326 ++++++++++++++++++++++++++++++---- fs/nfsd/nfs4state.c | 82 +++++++-- fs/nfsd/nfsctl.c | 1 + fs/nfsd/state.h | 8 +- include/uapi/linux/nfsd/cld.h | 1 + 6 files changed, 372 insertions(+), 49 deletions(-)Signature · 1 line
2.17.1