add support for CB_RECALL_ANY and the delegation shrinker
Subsystems (2)
Dai Ngo
reviewerv3 cover add support for CB_RECALL_ANY and the delegation shrinker
This patch series adds:
. support for sending the CB_RECALL_ANY op.
There is only one nfsd4_callback, cl_recall_any, added for each nfs4_client. Access to it must be serialized. For now it's done by the cl_recall_any_busy flag since it's used only by the delegation shrinker. If there is another consumer of cl_recall_any then a spinlock must be used.
. the delegation shrinker that sends the advisory CB_RECALL_ANY
to the clients to release unused delegations.
v2:
. modify deleg_reaper to check and send CB_RECALL_ANY to client
only once per 5 secs.
v3:
. modify nfsd4_cb_recall_any_release to use nn->client_lock to
protect cl_recall_any_busy and call put_client_renew_locked to decrement cl_rpc_users.
---
Dai Ngo (2):
NFSD: add support for sending CB_RECALL_ANY
NFSD: add delegation shrinker to react to low memory condition
fs/nfsd/netns.h | 3 ++ fs/nfsd/nfs4callback.c | 64 ++++++++++++++++++++++++++ fs/nfsd/nfs4state.c | 106 +++++++++++++++++++++++++++++++++++++++++++- fs/nfsd/state.h | 10 +++++ fs/nfsd/xdr4cb.h | 6 +++ 5 files changed, 188 insertions(+), 1 deletion(-)
Dai Ngo
reviewerPATCH v3 1/2 NFSD: add support for sending CB_RECALL_ANY
There is only one nfsd4_callback, cl_recall_any, added for each nfs4_client. Access to it must be serialized. For now it's done by the cl_recall_any_busy flag since it's used only by the delegation shrinker. If there is another consumer of CB_RECALL_ANY then a spinlock must be used.
--- fs/nfsd/nfs4callback.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ fs/nfsd/nfs4state.c | 32 +++++++++++++++++++++++++ fs/nfsd/state.h | 8 +++++++ fs/nfsd/xdr4cb.h | 6 +++++ 4 files changed, 110 insertions(+)
fs/nfsd/nfs4callback.c +64 -0
@@ -329,6 +329,29 @@ static void encode_cb_recall4args(struct xdr_stream *xdr, }/* +* CB_RECALLANY4args +* +* struct CB_RECALLANY4args { +* uint32_t craa_objects_to_keep; +* bitmap4 craa_type_mask; +* }; +*/ +static void +encode_cb_recallany4args (struct xdr_stream *xdr, +struct nfs4_cb_compound_hdr *hdr,uint32_t bmval) +{ +__be32 *p; + +encode_nfs_cb_opnum4 (xdr,OP_CB_RECALL_ANY ); + p =xdr_reserve_space (xdr,4 ); + *p++ = xdr_zero;/* craa_objects_to_keep */ + p =xdr_reserve_space (xdr,8 ); + *p++ =cpu_to_be32 (1 ); + *p++ =cpu_to_be32 (bmval); + hdr->nops++; +} + +/* * CB_SEQUENCE4args * * struct CB_SEQUENCE4args {
@@ -482,6 +505,24 @@ static void nfs4_xdr_enc_cb_recall(struct rpc_rqst *req, struct xdr_stream *xdr,encode_cb_nops (&hdr); } +/* +* 20.6. Operation 8: CB_RECALL_ANY - Keep Any N Recallable Objects +*/ +static void +nfs4_xdr_enc_cb_recall_any (struct rpc_rqst *req, +struct xdr_stream *xdr,const void *data) +{ +const struct nfsd4_callback *cb = data; +struct nfs4_cb_compound_hdr hdr = { + .ident = cb->cb_clp->cl_cb_ident, + .minorversion = cb->cb_clp->cl_minorversion, + }; + +encode_cb_compound4args (xdr, &hdr); +encode_cb_sequence4args (xdr, cb, &hdr); +encode_cb_recallany4args (xdr, &hdr, cb->cb_clp->cl_recall_any_bm); +encode_cb_nops (&hdr); +}/* * NFSv4.0 and NFSv4.1 XDR decode functions
@@ -520,6 +561,28 @@ static int nfs4_xdr_dec_cb_recall(struct rpc_rqst *rqstp,return decode_cb_op_status (xdr,OP_CB_RECALL , &cb->cb_status); } +/* +* 20.6. Operation 8: CB_RECALL_ANY - Keep Any N Recallable Objects +*/ +static int +nfs4_xdr_dec_cb_recall_any (struct rpc_rqst *rqstp, +struct xdr_stream *xdr, +void *data) +{ +struct nfsd4_callback *cb = data; +struct nfs4_cb_compound_hdr hdr; +int status; + + status =decode_cb_compound4res (xdr, &hdr); +if (unlikely (status)) +return status; + status =decode_cb_sequence4res (xdr, cb); +if (unlikely (status || cb->cb_seq_status)) +return status; + status =decode_cb_op_status (xdr,OP_CB_RECALL_ANY , &cb->cb_status); +return status; +} +#ifdef CONFIG_NFSD_PNFS /* * CB_LAYOUTRECALL4args
@@ -783,6 +846,7 @@ static const struct rpc_procinfo nfs4_cb_procedures[] = {#endif PROC (CB_NOTIFY_LOCK ,COMPOUND , cb_notify_lock, cb_notify_lock),PROC (CB_OFFLOAD ,COMPOUND , cb_offload, cb_offload), + PROC(CB_RECALL_ANY, COMPOUND, cb_recall_any, cb_recall_any), };static unsigned int nfs4_cb_counts[ARRAY_SIZE (nfs4_cb_procedures)];
fs/nfsd/nfs4state.c +32 -0
@@ -2854,6 +2854,36 @@ static const struct tree_descr client_files[] = { [3] = {""}, }; +static int +nfsd4_cb_recall_any_done (struct nfsd4_callback *cb, +struct rpc_task *task) +{ +switch (task->tk_status) { +case -NFS4ERR_DELAY : +rpc_delay (task,2 *HZ ); +return 0 ; +default : +return 1 ; + } +} + +static void +nfsd4_cb_recall_any_release (struct nfsd4_callback *cb) +{ +struct nfs4_client *clp = cb->cb_clp; +struct nfsd_net *nn =net_generic (clp->net, nfsd_net_id); + +spin_lock (&nn->client_lock); + clp->cl_recall_any_busy = false; +put_client_renew_locked (clp); +spin_unlock (&nn->client_lock); +} + +static const struct nfsd4_callback_ops nfsd4_cb_recall_any_ops = { + .done = nfsd4_cb_recall_any_done, + .release = nfsd4_cb_recall_any_release, +}; +static struct nfs4_client *create_client (struct xdr_netobj name,struct svc_rqst *rqstp,nfs4_verifier *verf) {
@@ -2891,6 +2921,8 @@ static struct nfs4_client *create_client(struct xdr_netobj name,free_client (clp);return NULL ; } +nfsd4_init_cb (&clp->cl_recall_any, clp, &nfsd4_cb_recall_any_ops, +NFSPROC4_CLNT_CB_RECALL_ANY );return clp; }
fs/nfsd/state.h +8 -0
@@ -411,6 +411,10 @@ struct nfs4_client {unsigned int cl_state;atomic_t cl_delegs_in_recall; + +bool cl_recall_any_busy; +uint32_t cl_recall_any_bm; +struct nfsd4_callback cl_recall_any; };/* struct nfs4_client_reset
@@ -639,8 +643,12 @@ enum nfsd4_cb_op {NFSPROC4_CLNT_CB_OFFLOAD ,NFSPROC4_CLNT_CB_SEQUENCE ,NFSPROC4_CLNT_CB_NOTIFY_LOCK , + NFSPROC4_CLNT_CB_RECALL_ANY, }; +#define RCA4_TYPE_MASK_RDATA_DLG 0 +#define RCA4_TYPE_MASK_WDATA_DLG 1 +/* Returns true iff a is later than b: */ static inline bool nfsd4_stateid_generation_after (stateid_t *a,stateid_t *b) {
fs/nfsd/xdr4cb.h +6 -0
@@ -48,3 +48,9 @@#define NFS4_dec_cb_offload_sz (cb_compound_dec_hdr_sz + \ cb_sequence_dec_sz + \ op_dec_sz) +#define NFS4_enc_cb_recall_any_sz (cb_compound_enc_hdr_sz + \ + cb_sequence_enc_sz + \ + 1 + 1 + 1) +#define NFS4_dec_cb_recall_any_sz (cb_compound_dec_hdr_sz + \ + cb_sequence_dec_sz + \ + op_dec_sz)
Signature · 1 line
2.9.5
Re: [PATCH v3 1/2] NFSD: add support for sending CB_RECALL_ANY
Quoted 6 lines · Dai Ngo
There is only one
nfsd4_callback,cl_recall_any, added for eachnfs4_client.
Is only one needed at a time, or might NFSD need to send more than one concurrently? So far it looks like the only reason to limit it to one at a time is the way the cb_recall_any arguments are passed to the XDR layer.
Quoted 4 lines
Access to it must be serialized. For now it's done
by thecl_recall_any_busyflag since it's used only by the delegation shrinker. If there is another consumer ofCB_RECALL_ANYthen a spinlock must be used.
The usual arrangement is to add the XDR infrastructure for a new operation in one patch, and then add consumers in subsequent patches. Can you move the hunks that change fs/nfsd/nfs4state.c to 2/2 and update the above description accordingly?
In a separate patch you should add a trace_nfsd_cb_recall_any and a trace_nfsd_cb_recall_any_done tracepoint. There are already nice examples in fs/nfsd/trace.h for the other callback operations.
A little more below.
Quoted 22 lines
Signed-off-by Dai Ngo <[email protected]>---
fs/nfsd/nfs4callback.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ fs/nfsd/nfs4state.c | 32 +++++++++++++++++++++++++ fs/nfsd/state.h | 8 +++++++ fs/nfsd/xdr4cb.h | 6 +++++ 4 files changed, 110 insertions(+)fs/nfsd/nfs4callback.c +6
-0@@ -329,6 +329,29 @@ static void encode_cb_recall4args(struct xdr_stream *xdr, }/* +* CB_RECALLANY4args +* +* struct CB_RECALLANY4args { +* uint32_t craa_objects_to_keep; +* bitmap4 craa_type_mask; +* };
fs/nfsd/nfs4callback.c static void encode_cb_recall4args(struct xdr_stream *xdr, · line 347
Quoted 10 lines
fs/nfsd/nfs4callback.c +10
-0+*/ +static void +encode_cb_recallany4args (struct xdr_stream *xdr, +struct nfs4_cb_compound_hdr *hdr,uint32_t bmval) +{ +__be32 *p; + +encode_nfs_cb_opnum4 (xdr,OP_CB_RECALL_ANY ); + p =xdr_reserve_space (xdr,4 ); + *p++ = xdr_zero;/* craa_objects_to_keep */
Let's use xdr_stream_encode_u32() here. Would it be reasonable for the upper layer to provide this value, or will NFSD always want it to be zero?
fs/nfsd/nfs4callback.c static void encode_cb_recall4args(struct xdr_stream *xdr, · line 348
Quoted 1 line
fs/nfsd/nfs4callback.c +1
-0+ p =xdr_reserve_space (xdr,8 );
Let's use xdr_stream_encode_uint32_array() here. encode_cb_recallany4args's caller should pass a u32 * and a length, not just a simple u32.
Quoted 126 lines
fs/nfsd/nfs4callback.c +47
-0+ *p++ =cpu_to_be32 (1 ); + *p++ =cpu_to_be32 (bmval); + hdr->nops++; +} + +/* * CB_SEQUENCE4args * * struct CB_SEQUENCE4args { @@ -482,6 +505,24 @@ static void nfs4_xdr_enc_cb_recall(struct rpc_rqst *req, struct xdr_stream *xdr,encode_cb_nops (&hdr); } +/* +* 20.6. Operation 8: CB_RECALL_ANY - Keep Any N Recallable Objects +*/ +static void +nfs4_xdr_enc_cb_recall_any (struct rpc_rqst *req, +struct xdr_stream *xdr,const void *data) +{ +const struct nfsd4_callback *cb = data; +struct nfs4_cb_compound_hdr hdr = { + .ident = cb->cb_clp->cl_cb_ident, + .minorversion = cb->cb_clp->cl_minorversion, + }; + +encode_cb_compound4args (xdr, &hdr); +encode_cb_sequence4args (xdr, cb, &hdr); +encode_cb_recallany4args (xdr, &hdr, cb->cb_clp->cl_recall_any_bm); +encode_cb_nops (&hdr); +}/* * NFSv4.0 and NFSv4.1 XDR decode functions @@ -520,6 +561,28 @@ static int nfs4_xdr_dec_cb_recall(struct rpc_rqst *rqstp,return decode_cb_op_status (xdr,OP_CB_RECALL , &cb->cb_status); } +/* +* 20.6. Operation 8: CB_RECALL_ANY - Keep Any N Recallable Objects +*/ +static int +nfs4_xdr_dec_cb_recall_any (struct rpc_rqst *rqstp, +struct xdr_stream *xdr, +void *data) +{ +struct nfsd4_callback *cb = data; +struct nfs4_cb_compound_hdr hdr; +int status; + + status =decode_cb_compound4res (xdr, &hdr); +if (unlikely (status)) +return status; + status =decode_cb_sequence4res (xdr, cb); +if (unlikely (status || cb->cb_seq_status)) +return status; + status =decode_cb_op_status (xdr,OP_CB_RECALL_ANY , &cb->cb_status); +return status; +} +#ifdef CONFIG_NFSD_PNFS /* * CB_LAYOUTRECALL4args @@ -783,6 +846,7 @@ static const struct rpc_procinfo nfs4_cb_procedures[] = {#endif PROC (CB_NOTIFY_LOCK ,COMPOUND , cb_notify_lock, cb_notify_lock),PROC (CB_OFFLOAD ,COMPOUND , cb_offload, cb_offload), + PROC(CB_RECALL_ANY, COMPOUND, cb_recall_any, cb_recall_any), };static unsigned int nfs4_cb_counts[ARRAY_SIZE (nfs4_cb_procedures)];fs/nfsd/nfs4state.c +32
-0@@ -2854,6 +2854,36 @@ static const struct tree_descr client_files[] = { [3] = {""}, }; +static int +nfsd4_cb_recall_any_done (struct nfsd4_callback *cb, +struct rpc_task *task) +{ +switch (task->tk_status) { +case -NFS4ERR_DELAY : +rpc_delay (task,2 *HZ ); +return 0 ; +default : +return 1 ; + } +} + +static void +nfsd4_cb_recall_any_release (struct nfsd4_callback *cb) +{ +struct nfs4_client *clp = cb->cb_clp; +struct nfsd_net *nn =net_generic (clp->net, nfsd_net_id); + +spin_lock (&nn->client_lock); + clp->cl_recall_any_busy = false; +put_client_renew_locked (clp); +spin_unlock (&nn->client_lock); +} + +static const struct nfsd4_callback_ops nfsd4_cb_recall_any_ops = { + .done = nfsd4_cb_recall_any_done, + .release = nfsd4_cb_recall_any_release, +}; +static struct nfs4_client *create_client (struct xdr_netobj name,struct svc_rqst *rqstp,nfs4_verifier *verf) {@@ -2891,6 +2921,8 @@ static struct nfs4_client *create_client(struct xdr_netobj name,free_client (clp);return NULL ; } +nfsd4_init_cb (&clp->cl_recall_any, clp, &nfsd4_cb_recall_any_ops, +NFSPROC4_CLNT_CB_RECALL_ANY );return clp; }fs/nfsd/state.h +0
-0
fs/nfsd/state.h struct nfs4_client { · line 415
Quoted 5 lines
fs/nfsd/state.h +2
-0@@ -411,6 +411,10 @@ struct nfs4_client {unsigned int cl_state;atomic_t cl_delegs_in_recall; + +bool cl_recall_any_busy;
Rather than adding a boolean field, you could add a bit to cl_flags.
I'm not convinced you need to add the argument fields here... I think kmalloc'ing the arguments and then freeing them in nfsd4_cb_recall_any_release() would be sufficient.
Quoted 35 lines
fs/nfsd/state.h +6
-0+uint32_t cl_recall_any_bm; +struct nfsd4_callback cl_recall_any; };/* struct nfs4_client_reset @@ -639,8 +643,12 @@ enum nfsd4_cb_op {NFSPROC4_CLNT_CB_OFFLOAD ,NFSPROC4_CLNT_CB_SEQUENCE ,NFSPROC4_CLNT_CB_NOTIFY_LOCK , + NFSPROC4_CLNT_CB_RECALL_ANY, }; +#define RCA4_TYPE_MASK_RDATA_DLG 0 +#define RCA4_TYPE_MASK_WDATA_DLG 1 +/* Returns true iff a is later than b: */ static inline bool nfsd4_stateid_generation_after (stateid_t *a,stateid_t *b) {fs/nfsd/xdr4cb.h +6
-0@@ -48,3 +48,9 @@#define NFS4_dec_cb_offload_sz (cb_compound_dec_hdr_sz + \ cb_sequence_dec_sz + \ op_dec_sz) +#define NFS4_enc_cb_recall_any_sz (cb_compound_enc_hdr_sz + \ + cb_sequence_enc_sz + \ + 1 + 1 + 1) +#define NFS4_dec_cb_recall_any_sz (cb_compound_dec_hdr_sz + \ + cb_sequence_dec_sz + \ + op_dec_sz)Signature · 1 line
2.9.5
--
Chuck Lever
[email protected]
reviewerRe: [PATCH v3 1/2] NFSD: add support for sending CB_RECALL_ANY
Quoted 9 lines · Chuck Lever III
Quoted 5 lines · Dai Ngo
There is only one
nfsd4_callback,cl_recall_any, added for eachnfs4_client.Is only one needed at a time, or might NFSD need to send more than one concurrently? So far it looks like the only reason to limit it to one at a time is the way the
cb_recall_anyarguments are passed to the XDR layer.
We only need to send one CB_RECALL_ANY for each client that hold delegations so just one nfsd4_callback needed for this purpose. Do you see a need to support multiple nfsd4_callback's per client for cl_recall_any?
Quoted 10 lines
Quoted 6 lines
Access to it must be serialized. For now it's done
by thecl_recall_any_busyflag since it's used only by the delegation shrinker. If there is another consumer ofCB_RECALL_ANYthen a spinlock must be used.The usual arrangement is to add the XDR infrastructure for a new operation in one patch, and then add consumers in subsequent patches. Can you move the hunks that change
fs/nfsd/nfs4state.cto 2/2 and update the above description accordingly?
fix in v4.
Quoted 4 lines
In a separate patch you should add a
trace_nfsd_cb_recall_anyand atrace_nfsd_cb_recall_any_donetracepoint. There are already nice examples infs/nfsd/trace.hfor the other callback operations.
fix in v4.
Quoted 30 lines
A little more below.
Quoted 25 lines
Signed-off-by Dai Ngo <[email protected]>---
fs/nfsd/nfs4callback.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ fs/nfsd/nfs4state.c | 32 +++++++++++++++++++++++++ fs/nfsd/state.h | 8 +++++++ fs/nfsd/xdr4cb.h | 6 +++++ 4 files changed, 110 insertions(+)fs/nfsd/nfs4callback.c +9
-0@@ -329,6 +329,29 @@ static void encode_cb_recall4args(struct xdr_stream *xdr, }/* +* CB_RECALLANY4args +* +* struct CB_RECALLANY4args { +* uint32_t craa_objects_to_keep; +* bitmap4 craa_type_mask; +* }; +*/ +static void +encode_cb_recallany4args(struct xdr_stream *xdr,
Quoted 8 lines
fs/nfsd/nfs4callback.c static void encode_cb_recall4args(struct xdr_stream *xdr, · line 347
Quoted 7 lines
fs/nfsd/nfs4callback.c +7
-0+ struct nfs4_cb_compound_hdr *hdr, uint32_t bmval) +{ +__be32 *p; + +encode_nfs_cb_opnum4 (xdr,OP_CB_RECALL_ANY ); + p =xdr_reserve_space (xdr,4 ); + *p++ = xdr_zero;/* craa_objects_to_keep */ Let's use
xdr_stream_encode_u32()here.
Yes, we can use xdr_stream_encode_u32() here. However xdr_stream_encode_u32 can return error and currently all the xdr encoding functions, nfs4_xdr_enc_xxxx, are defined to return void so rpcauth_wrap_req_encode just ignores encode errors always return 0. Note that gss_wrap_req_integ and gss_wrap_req_priv actually return encoding errors.
Quoted 3 lines
Would it be reasonable
for the upper layer to provide this value, or will NFSD always want it to be zero?
Ideally the XDR encode routine should not make any decision regarding how many objects the client can keep, it's up to the consumer of the CB_RECALL_ANY to decide. However in this case, NFSD always want to set this to 0 so instead of passing this in as another argument to the encoder, I just did the short cut here. Do you want to pass this in as an argument?
Quoted 5 lines
fs/nfsd/nfs4callback.c static void encode_cb_recall4args(struct xdr_stream *xdr, · line 348
Quoted 3 lines
fs/nfsd/nfs4callback.c +1
-0+ p =xdr_reserve_space (xdr,8 );Let's use
xdr_stream_encode_uint32_array()here. encode_cb_recallany4args's caller should pass a u32 * and a length, not just a simple u32.
fix in v4.
Quoted 129 lines
Quoted 129 lines
fs/nfsd/nfs4callback.c +47
-0+ *p++ =cpu_to_be32 (1 ); + *p++ =cpu_to_be32 (bmval); + hdr->nops++; +} + +/* * CB_SEQUENCE4args * * struct CB_SEQUENCE4args { @@ -482,6 +505,24 @@ static void nfs4_xdr_enc_cb_recall(struct rpc_rqst *req, struct xdr_stream *xdr,encode_cb_nops (&hdr); } +/* +* 20.6. Operation 8: CB_RECALL_ANY - Keep Any N Recallable Objects +*/ +static void +nfs4_xdr_enc_cb_recall_any (struct rpc_rqst *req, +struct xdr_stream *xdr,const void *data) +{ +const struct nfsd4_callback *cb = data; +struct nfs4_cb_compound_hdr hdr = { + .ident = cb->cb_clp->cl_cb_ident, + .minorversion = cb->cb_clp->cl_minorversion, + }; + +encode_cb_compound4args (xdr, &hdr); +encode_cb_sequence4args (xdr, cb, &hdr); +encode_cb_recallany4args (xdr, &hdr, cb->cb_clp->cl_recall_any_bm); +encode_cb_nops (&hdr); +}/* * NFSv4.0 and NFSv4.1 XDR decode functions @@ -520,6 +561,28 @@ static int nfs4_xdr_dec_cb_recall(struct rpc_rqst *rqstp,return decode_cb_op_status (xdr,OP_CB_RECALL , &cb->cb_status); } +/* +* 20.6. Operation 8: CB_RECALL_ANY - Keep Any N Recallable Objects +*/ +static int +nfs4_xdr_dec_cb_recall_any (struct rpc_rqst *rqstp, +struct xdr_stream *xdr, +void *data) +{ +struct nfsd4_callback *cb = data; +struct nfs4_cb_compound_hdr hdr; +int status; + + status =decode_cb_compound4res (xdr, &hdr); +if (unlikely (status)) +return status; + status =decode_cb_sequence4res (xdr, cb); +if (unlikely (status || cb->cb_seq_status)) +return status; + status =decode_cb_op_status (xdr,OP_CB_RECALL_ANY , &cb->cb_status); +return status; +} +#ifdef CONFIG_NFSD_PNFS /* * CB_LAYOUTRECALL4args @@ -783,6 +846,7 @@ static const struct rpc_procinfo nfs4_cb_procedures[] = {#endif PROC (CB_NOTIFY_LOCK ,COMPOUND , cb_notify_lock, cb_notify_lock),PROC (CB_OFFLOAD ,COMPOUND , cb_offload, cb_offload), + PROC(CB_RECALL_ANY, COMPOUND, cb_recall_any, cb_recall_any), };static unsigned int nfs4_cb_counts[ARRAY_SIZE (nfs4_cb_procedures)];fs/nfsd/nfs4state.c +32
-0@@ -2854,6 +2854,36 @@ static const struct tree_descr client_files[] = { [3] = {""}, }; +static int +nfsd4_cb_recall_any_done (struct nfsd4_callback *cb, +struct rpc_task *task) +{ +switch (task->tk_status) { +case -NFS4ERR_DELAY : +rpc_delay (task,2 *HZ ); +return 0 ; +default : +return 1 ; + } +} + +static void +nfsd4_cb_recall_any_release (struct nfsd4_callback *cb) +{ +struct nfs4_client *clp = cb->cb_clp; +struct nfsd_net *nn =net_generic (clp->net, nfsd_net_id); + +spin_lock (&nn->client_lock); + clp->cl_recall_any_busy = false; +put_client_renew_locked (clp); +spin_unlock (&nn->client_lock); +} + +static const struct nfsd4_callback_ops nfsd4_cb_recall_any_ops = { + .done = nfsd4_cb_recall_any_done, + .release = nfsd4_cb_recall_any_release, +}; +static struct nfs4_client *create_client (struct xdr_netobj name,struct svc_rqst *rqstp,nfs4_verifier *verf) {@@ -2891,6 +2921,8 @@ static struct nfs4_client *create_client(struct xdr_netobj name,free_client (clp);return NULL ; } +nfsd4_init_cb (&clp->cl_recall_any, clp, &nfsd4_cb_recall_any_ops, +NFSPROC4_CLNT_CB_RECALL_ANY );return clp; }fs/nfsd/state.h +0
-0@@ -411,6 +411,10 @@ struct nfs4_client {
Quoted 6 lines
fs/nfsd/state.h struct nfs4_client { · line 415
Quoted 4 lines
fs/nfsd/state.h +2
-0unsigned int cl_state;atomic_t cl_delegs_in_recall; + +bool cl_recall_any_busy;Rather than adding a boolean field, you could add a bit to
cl_flags.
fix in v4.
Quoted 4 lines
I'm not convinced you need to add the argument fields here... I think kmalloc'ing the arguments and then freeing them in
nfsd4_cb_recall_any_release()would be sufficient.
Since cb_recall_any is sent when we're running low on system memory, I'm trying not to use kmalloc'ing to avoid potential deadlock or adding more stress to the system.
Thanks!
-Dai
Quoted 42 lines
Quoted 37 lines
fs/nfsd/state.h +6
-0+uint32_t cl_recall_any_bm; +struct nfsd4_callback cl_recall_any; };/* struct nfs4_client_reset @@ -639,8 +643,12 @@ enum nfsd4_cb_op {NFSPROC4_CLNT_CB_OFFLOAD ,NFSPROC4_CLNT_CB_SEQUENCE ,NFSPROC4_CLNT_CB_NOTIFY_LOCK , + NFSPROC4_CLNT_CB_RECALL_ANY, }; +#define RCA4_TYPE_MASK_RDATA_DLG 0 +#define RCA4_TYPE_MASK_WDATA_DLG 1 +/* Returns true iff a is later than b: */ static inline bool nfsd4_stateid_generation_after (stateid_t *a,stateid_t *b) {fs/nfsd/xdr4cb.h +6
-0@@ -48,3 +48,9 @@#define NFS4_dec_cb_offload_sz (cb_compound_dec_hdr_sz + \ cb_sequence_dec_sz + \ op_dec_sz) +#define NFS4_enc_cb_recall_any_sz (cb_compound_enc_hdr_sz + \ + cb_sequence_enc_sz + \ + 1 + 1 + 1) +#define NFS4_dec_cb_recall_any_sz (cb_compound_dec_hdr_sz + \ + cb_sequence_dec_sz + \ + op_dec_sz)Signature · 1 line
2.9.5--
Chuck Lever
Re: [PATCH v3 1/2] NFSD: add support for sending CB_RECALL_ANY
Quoted 15 lines · Dai Ngo
Quoted 9 lines · Chuck Lever III
Quoted 5 lines · Dai Ngo
There is only one
nfsd4_callback,cl_recall_any, added for eachnfs4_client.Is only one needed at a time, or might NFSD need to send more than one concurrently? So far it looks like the only reason to limit it to one at a time is the way the
cb_recall_anyarguments are passed to the XDR layer.
Quoted 4 lines
We only need to send one
CB_RECALL_ANYfor each client that hold delegations so just onenfsd4_callbackneeded for this purpose. Do you see a need to support multiple nfsd4_callback's per client forcl_recall_any?
CB_RECALL_ANY has a bitmap argument so that it can be used for recalling many different types of state. I don't pretend to understand all of it, but it seems like eventually NFSD might want to send more than one of these at a time, and not just in low memory scenarios. I suggest a way below of doing this without adding complexity.
Quoted 57 lines
Quoted 8 lines
Quoted 4 lines
Access to it must be serialized. For now it's done
by thecl_recall_any_busyflag since it's used only by the delegation shrinker. If there is another consumer ofCB_RECALL_ANYthen a spinlock must be used.The usual arrangement is to add the XDR infrastructure for a new operation in one patch, and then add consumers in subsequent patches. Can you move the hunks that change
fs/nfsd/nfs4state.cto 2/2 and update the above description accordingly?fix in v4.
Quoted 4 lines
In a separate patch you should add a
trace_nfsd_cb_recall_anyand atrace_nfsd_cb_recall_any_donetracepoint. There are already nice examples infs/nfsd/trace.hfor the other callback operations.fix in v4.
Quoted 30 lines
A little more below.
Quoted 25 lines
Signed-off-by Dai Ngo <[email protected]>---
fs/nfsd/nfs4callback.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ fs/nfsd/nfs4state.c | 32 +++++++++++++++++++++++++ fs/nfsd/state.h | 8 +++++++ fs/nfsd/xdr4cb.h | 6 +++++ 4 files changed, 110 insertions(+)fs/nfsd/nfs4callback.c +9
-0@@ -329,6 +329,29 @@ static void encode_cb_recall4args(struct xdr_stream *xdr, }/* +* CB_RECALLANY4args +* +* struct CB_RECALLANY4args { +* uint32_t craa_objects_to_keep; +* bitmap4 craa_type_mask; +* }; +*/ +static void +encode_cb_recallany4args(struct xdr_stream *xdr,Quoted 8 lines
fs/nfsd/nfs4callback.c static void encode_cb_recall4args(struct xdr_stream *xdr, · line 347
Quoted 7 lines
fs/nfsd/nfs4callback.c +7
-0+ struct nfs4_cb_compound_hdr *hdr, uint32_t bmval) +{ +__be32 *p; + +encode_nfs_cb_opnum4 (xdr,OP_CB_RECALL_ANY ); + p =xdr_reserve_space (xdr,4 ); + *p++ = xdr_zero;/* craa_objects_to_keep */ Let's use
xdr_stream_encode_u32()here.
Quoted 5 lines
Yes, we can use
xdr_stream_encode_u32()here. Howeverxdr_stream_encode_u32can return error and currently all the xdr encoding functions,nfs4_xdr_enc_xxxx, are defined to return void sorpcauth_wrap_req_encodejust ignores encode errors always return 0. Note thatgss_wrap_req_integandgss_wrap_req_privactually return encoding errors.
The convention it seems is to ignore errors in this code and assume that the upper layer has set up the send buffer properly. Anything else would be a local software bug, so it is probably a safe assumption.
Quoted 8 lines
Quoted 3 lines
Would it be reasonable
for the upper layer to provide this value, or will NFSD always want it to be zero?Ideally the XDR encode routine should not make any decision regarding how many objects the client can keep, it's up to the consumer of the
CB_RECALL_ANYto decide. However in this case, NFSD always want to set this to 0
"in this case" I assume means in the low-memory scenario. I don't know enough to say if other scenarios might want to choose a different value.
Quoted 3 lines
so instead of passing this in as another argument to the encoder, I just did the short cut here. Do you want to pass this in as an argument?
I don't think it would be difficult to parametrize this, so give it a shot. Suggestion way below.
Quoted 145 lines
Quoted 3 lines
fs/nfsd/nfs4callback.c static void encode_cb_recall4args(struct xdr_stream *xdr, · line 348
Quoted 1 line
fs/nfsd/nfs4callback.c +1
-0+ p =xdr_reserve_space (xdr,8 );Let's use
xdr_stream_encode_uint32_array()here. encode_cb_recallany4args's caller should pass a u32 * and a length, not just a simple u32.fix in v4.
Quoted 129 lines
Quoted 129 lines
fs/nfsd/nfs4callback.c +47
-0+ *p++ =cpu_to_be32 (1 ); + *p++ =cpu_to_be32 (bmval); + hdr->nops++; +} + +/* * CB_SEQUENCE4args * * struct CB_SEQUENCE4args { @@ -482,6 +505,24 @@ static void nfs4_xdr_enc_cb_recall(struct rpc_rqst *req, struct xdr_stream *xdr,encode_cb_nops (&hdr); } +/* +* 20.6. Operation 8: CB_RECALL_ANY - Keep Any N Recallable Objects +*/ +static void +nfs4_xdr_enc_cb_recall_any (struct rpc_rqst *req, +struct xdr_stream *xdr,const void *data) +{ +const struct nfsd4_callback *cb = data; +struct nfs4_cb_compound_hdr hdr = { + .ident = cb->cb_clp->cl_cb_ident, + .minorversion = cb->cb_clp->cl_minorversion, + }; + +encode_cb_compound4args (xdr, &hdr); +encode_cb_sequence4args (xdr, cb, &hdr); +encode_cb_recallany4args (xdr, &hdr, cb->cb_clp->cl_recall_any_bm); +encode_cb_nops (&hdr); +}/* * NFSv4.0 and NFSv4.1 XDR decode functions @@ -520,6 +561,28 @@ static int nfs4_xdr_dec_cb_recall(struct rpc_rqst *rqstp,return decode_cb_op_status (xdr,OP_CB_RECALL , &cb->cb_status); } +/* +* 20.6. Operation 8: CB_RECALL_ANY - Keep Any N Recallable Objects +*/ +static int +nfs4_xdr_dec_cb_recall_any (struct rpc_rqst *rqstp, +struct xdr_stream *xdr, +void *data) +{ +struct nfsd4_callback *cb = data; +struct nfs4_cb_compound_hdr hdr; +int status; + + status =decode_cb_compound4res (xdr, &hdr); +if (unlikely (status)) +return status; + status =decode_cb_sequence4res (xdr, cb); +if (unlikely (status || cb->cb_seq_status)) +return status; + status =decode_cb_op_status (xdr,OP_CB_RECALL_ANY , &cb->cb_status); +return status; +} +#ifdef CONFIG_NFSD_PNFS /* * CB_LAYOUTRECALL4args @@ -783,6 +846,7 @@ static const struct rpc_procinfo nfs4_cb_procedures[] = {#endif PROC (CB_NOTIFY_LOCK ,COMPOUND , cb_notify_lock, cb_notify_lock),PROC (CB_OFFLOAD ,COMPOUND , cb_offload, cb_offload), + PROC(CB_RECALL_ANY, COMPOUND, cb_recall_any, cb_recall_any), };static unsigned int nfs4_cb_counts[ARRAY_SIZE (nfs4_cb_procedures)];fs/nfsd/nfs4state.c +32
-0@@ -2854,6 +2854,36 @@ static const struct tree_descr client_files[] = { [3] = {""}, }; +static int +nfsd4_cb_recall_any_done (struct nfsd4_callback *cb, +struct rpc_task *task) +{ +switch (task->tk_status) { +case -NFS4ERR_DELAY : +rpc_delay (task,2 *HZ ); +return 0 ; +default : +return 1 ; + } +} + +static void +nfsd4_cb_recall_any_release (struct nfsd4_callback *cb) +{ +struct nfs4_client *clp = cb->cb_clp; +struct nfsd_net *nn =net_generic (clp->net, nfsd_net_id); + +spin_lock (&nn->client_lock); + clp->cl_recall_any_busy = false; +put_client_renew_locked (clp); +spin_unlock (&nn->client_lock); +} + +static const struct nfsd4_callback_ops nfsd4_cb_recall_any_ops = { + .done = nfsd4_cb_recall_any_done, + .release = nfsd4_cb_recall_any_release, +}; +static struct nfs4_client *create_client (struct xdr_netobj name,struct svc_rqst *rqstp,nfs4_verifier *verf) {@@ -2891,6 +2921,8 @@ static struct nfs4_client *create_client(struct xdr_netobj name,free_client (clp);return NULL ; } +nfsd4_init_cb (&clp->cl_recall_any, clp, &nfsd4_cb_recall_any_ops, +NFSPROC4_CLNT_CB_RECALL_ANY );return clp; }fs/nfsd/state.h +0
-0@@ -411,6 +411,10 @@ struct nfs4_client {Quoted 6 lines
fs/nfsd/state.h struct nfs4_client { · line 415
Quoted 4 lines
fs/nfsd/state.h +2
-0unsigned int cl_state;atomic_t cl_delegs_in_recall; + +bool cl_recall_any_busy;Rather than adding a boolean field, you could add a bit to
cl_flags.fix in v4.
Quoted 1 line
Quoted 7 lines
Quoted 3 lines
I'm not convinced you need to add the argument fields here... I think kmalloc'ing the arguments and then freeing them in
nfsd4_cb_recall_any_release()would be sufficient.Since
cb_recall_anyis sent when we're running low on system memory, I'm trying not to use kmalloc'ing to avoid potential deadlock or adding more stress to the system.
Fair. Well, again, there are potentially other use cases that aren't concerned about memory pressure.
Construct something like this (in fs/nfsd/xdr4.h):
struct nfsd4_cb_recall_any {
struct nfsd4_callback ra_cb;
u32 ra_keep;
u32 ra_bmval[1];
};Then either add it to another long-term structure like nfs4_client or the upper layer can kmalloc it on demand. That way the XDR code supports both ways and does not have to care which the upper layer wants to use.
Quoted 42 lines
Thanks!
-DaiQuoted 39 lines
Quoted 37 lines
fs/nfsd/state.h +6
-0+uint32_t cl_recall_any_bm; +struct nfsd4_callback cl_recall_any; };/* struct nfs4_client_reset @@ -639,8 +643,12 @@ enum nfsd4_cb_op {NFSPROC4_CLNT_CB_OFFLOAD ,NFSPROC4_CLNT_CB_SEQUENCE ,NFSPROC4_CLNT_CB_NOTIFY_LOCK , + NFSPROC4_CLNT_CB_RECALL_ANY, }; +#define RCA4_TYPE_MASK_RDATA_DLG 0 +#define RCA4_TYPE_MASK_WDATA_DLG 1 +/* Returns true iff a is later than b: */ static inline bool nfsd4_stateid_generation_after (stateid_t *a,stateid_t *b) {fs/nfsd/xdr4cb.h +6
-0@@ -48,3 +48,9 @@#define NFS4_dec_cb_offload_sz (cb_compound_dec_hdr_sz + \ cb_sequence_dec_sz + \ op_dec_sz) +#define NFS4_enc_cb_recall_any_sz (cb_compound_enc_hdr_sz + \ + cb_sequence_enc_sz + \ + 1 + 1 + 1) +#define NFS4_dec_cb_recall_any_sz (cb_compound_dec_hdr_sz + \ + cb_sequence_dec_sz + \ + op_dec_sz)Signature · 1 line
2.9.5--
Chuck Lever
--
Chuck Lever
Dai Ngo
reviewerPATCH v3 2/2 NFSD: add delegation shrinker to react to low memory condition
In reply to Dai NgoThe delegation shrinker is scheduled to run on every shrinker's 'count' callback. It scans the client list and sends the courtesy CB_RECALL_ANY to the clients that hold delegations.
To avoid flooding the clients with CB_RECALL_ANY requests, the delegation shrinker is scheduled to run after a 5 seconds delay.
--- fs/nfsd/netns.h | 3 +++ fs/nfsd/nfs4state.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++- fs/nfsd/state.h | 2 ++ 3 files changed, 78 insertions(+), 1 deletion(-)
fs/nfsd/netns.h +3 -0
@@ -196,6 +196,9 @@ struct nfsd_net {atomic_t nfsd_courtesy_clients;struct shrinker nfsd_client_shrinker;struct delayed_work nfsd_shrinker_work; + +struct shrinker nfsd_deleg_shrinker; +struct delayed_work nfsd_deleg_shrinker_work; };/* Simple check to find out if a given net was properly initialized */
fs/nfsd/nfs4state.c +73 -1
115 lines not on this page · raw message
fs/nfsd/state.h +2 -0
@@ -415,6 +415,8 @@ struct nfs4_client {bool cl_recall_any_busy;uint32_t cl_recall_any_bm;struct nfsd4_callback cl_recall_any; +struct list_head cl_recall_any_cblist; +time64_t cl_recall_any_time; };/* struct nfs4_client_reset
Signature · 1 line
2.9.5