nfsd: Fix races between nfsd4_cb_release() and nfsd4_shutdown_callback()
Subsystems (2)
PATCH v2 1/1 nfsd: Fix races between nfsd4_cb_release() and nfsd4_shutdown_callback()
When we're destroying the client lease, and we call nfsd4_shutdown_callback(), we must ensure that we do not return before all outstanding callbacks have terminated and have released their payloads.
---
v2 - Fix a leak of clp->cl_cb_inflight when running null probes
fs/nfsd/nfs4callback.c | 97 +++++++++++++++++++++++++++++++++--------- fs/nfsd/state.h | 1 + 2 files changed, 79 insertions(+), 19 deletions(-)
fs/nfsd/nfs4callback.c +78 -19
@@ -826,6 +826,31 @@ static int max_cb_time(struct net *net)return max (nn->nfsd4_lease/10 , (time_t )1 ) *HZ ; } +static struct workqueue_struct *callback_wq; + +static bool nfsd4_queue_cb (struct nfsd4_callback *cb) +{ +return queue_work (callback_wq, &cb->cb_work); +} + +static void nfsd41_cb_inflight_begin (struct nfs4_client *clp) +{ +atomic_inc (&clp->cl_cb_inflight); +} + +static void nfsd41_cb_inflight_end (struct nfs4_client *clp) +{ + +if (atomic_dec_and_test (&clp->cl_cb_inflight)) +wake_up_var (&clp->cl_cb_inflight); +} + +static void nfsd41_cb_inflight_wait_complete (struct nfs4_client *clp) +{ +wait_var_event (&clp->cl_cb_inflight, + !atomic_read (&clp->cl_cb_inflight)); +} +static const struct cred *get_backchannel_cred (struct nfs4_client *clp,struct rpc_clnt *client,struct nfsd4_session *ses) {if (clp->cl_minorversion ==0 ) {
@@ -937,14 +962,21 @@ static void nfsd4_cb_probe_done(struct rpc_task *task, void *calldata) clp->cl_cb_state =NFSD4_CB_UP ; } +static void nfsd4_cb_probe_release (void *calldata) +{ + struct nfs4_client *clp = container_of(calldata, struct nfs4_client, cl_cb_null); + +nfsd41_cb_inflight_end (clp); + +} +static const struct rpc_call_ops nfsd4_cb_probe_ops = {/* XXX: release method to ensure we set the cb channel down if * necessary on early failure? */ .rpc_call_done = nfsd4_cb_probe_done, + .rpc_release = nfsd4_cb_probe_release, }; -static struct workqueue_struct *callback_wq; -/* * Poke the callback thread to process any updates to the callback * parameters, and send a null probe.
@@ -975,9 +1007,12 @@ void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *conn)* If the slot is available, then mark it busy. Otherwise, set the * thread for sleeping on the callback RPC wait queue. */ -static bool nfsd41_cb_get_slot (struct nfs4_client *clp,struct rpc_task *task) +static bool nfsd41_cb_get_slot (struct nfsd4_callback *cb,struct rpc_task *task) { -if (test_and_set_bit (0 , &clp->cl_cb_slot_busy) !=0 ) { +struct nfs4_client *clp = cb->cb_clp; + +if (!cb->cb_holds_slot && +test_and_set_bit (0 , &clp->cl_cb_slot_busy) !=0 ) {rpc_sleep_on (&clp->cl_cb_waitq, task,NULL );/* Race breaker */ if (test_and_set_bit (0 , &clp->cl_cb_slot_busy) !=0 ) {
@@ -985,10 +1020,32 @@ static bool nfsd41_cb_get_slot(struct nfs4_client *clp, struct rpc_task *task)return false; }rpc_wake_up_queued_task (&clp->cl_cb_waitq, task); + cb->cb_holds_slot = true; }return true; } +static void nfsd41_cb_release_slot (struct nfsd4_callback *cb) +{ +struct nfs4_client *clp = cb->cb_clp; + +if (cb->cb_holds_slot) { + cb->cb_holds_slot = false; +clear_bit (0 , &clp->cl_cb_slot_busy); +rpc_wake_up_next (&clp->cl_cb_waitq); + } +} + +static void nfsd41_destroy_cb (struct nfsd4_callback *cb) +{ +struct nfs4_client *clp = cb->cb_clp; + +nfsd41_cb_release_slot (cb); +if (cb->cb_ops && cb->cb_ops->release) + cb->cb_ops->release (cb); +nfsd41_cb_inflight_end (clp); +} +/* * TODO: cb_sequence should support referring call lists, cachethis, multiple * slots, and mark callback channel down on communication errors.
@@ -1005,11 +1062,8 @@ static void nfsd4_cb_prepare(struct rpc_task *task, void *calldata)*/ cb->cb_seq_status =1 ; cb->cb_status =0 ; -if (minorversion) { -if (!cb->cb_holds_slot && !nfsd41_cb_get_slot (clp, task)) -return ; - cb->cb_holds_slot = true; - } +if (minorversion && !nfsd41_cb_get_slot (cb, task)) +return ;rpc_call_start (task); }
@@ -1076,9 +1130,7 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback cb->cb_seq_status); } - cb->cb_holds_slot = false; -clear_bit (0 , &clp->cl_cb_slot_busy); -rpc_wake_up_next (&clp->cl_cb_waitq); +nfsd41_cb_release_slot (cb);dprintk ("%s: freed slot, new seqid=%d\n" , __func__, clp->cl_cb_session->se_cb_seq_nr);
@@ -1091,8 +1143,10 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback ret = false; goto out; need_restart: - task->tk_status =0 ; - cb->cb_need_restart = true; +if (!test_bit (NFSD4_CLIENT_CB_KILL , &clp->cl_flags)) { + task->tk_status =0 ; + cb->cb_need_restart = true; + }return false; }
@@ -1134,9 +1188,9 @@ static void nfsd4_cb_release(void *calldata)struct nfsd4_callback *cb = calldata;if (cb->cb_need_restart) -nfsd4_run_cb (cb); +nfsd4_queue_cb (cb);else - cb->cb_ops->release (cb); +nfsd41_destroy_cb (cb); }
@@ -1170,6 +1224,7 @@ void nfsd4_shutdown_callback(struct nfs4_client *clp)*/ nfsd4_run_cb (&clp->cl_cb_null);flush_workqueue (callback_wq); +nfsd41_cb_inflight_wait_complete (clp); }/* requires cl_lock: */
@@ -1255,8 +1310,7 @@ nfsd4_run_cb_work(struct work_struct *work) clnt = clp->cl_cb_client;if (!clnt) {/* Callback channel broken, or client killed; give up: */ -if (cb->cb_ops && cb->cb_ops->release) - cb->cb_ops->release (cb); +nfsd41_destroy_cb (cb);return ; }
@@ -1265,6 +1319,7 @@ nfsd4_run_cb_work(struct work_struct *work)*/ if (!cb->cb_ops && clp->cl_minorversion) { clp->cl_cb_state =NFSD4_CB_UP ; +nfsd41_destroy_cb (cb);return ; }
@@ -1290,5 +1345,9 @@ void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,void nfsd4_run_cb (struct nfsd4_callback *cb) { -queue_work (callback_wq, &cb->cb_work); +struct nfs4_client *clp = cb->cb_clp; + +nfsd41_cb_inflight_begin (clp); +if (!nfsd4_queue_cb (cb)) +nfsd41_cb_inflight_end (clp); }
fs/nfsd/state.h +1 -0
@@ -367,6 +367,7 @@ struct nfs4_client {struct net *net;struct list_head async_copies;/* list of async copies */ spinlock_t async_lock;/* lock for async copies */ +atomic_t cl_cb_inflight;/* Outstanding callbacks */ };/* struct nfs4_client_reset
Signature · 1 line
2.21.0
Quoted 4 lines · Trond Myklebust
When we're destroying the client lease, and we call
nfsd4_shutdown_callback(), we must ensure that we do not return before all outstanding callbacks have terminated and have released their payloads.
This is great, thanks! We've seen what I'm fairly sure is the same bug from Red Hat users. I think my blind spot was an assumption that rpc tasks wouldn't outlive rpc_shutdown_client().
However, it's causing xfstests runs to hang, and I haven't worked out why yet.
I'll spend some time on it this afternoon and let you know what I figure out.
--b.
Quoted 218 lines
Signed-off-by Trond Myklebust <[email protected]>---
v2 - Fix a leak ofclp->cl_cb_inflightwhen running null probesfs/nfsd/nfs4callback.c | 97 +++++++++++++++++++++++++++++++++--------- fs/nfsd/state.h | 1 + 2 files changed, 79 insertions(+), 19 deletions(-)fs/nfsd/nfs4callback.c +78
-19@@ -826,6 +826,31 @@ static int max_cb_time(struct net *net)return max (nn->nfsd4_lease/10 , (time_t )1 ) *HZ ; } +static struct workqueue_struct *callback_wq; + +static bool nfsd4_queue_cb (struct nfsd4_callback *cb) +{ +return queue_work (callback_wq, &cb->cb_work); +} + +static void nfsd41_cb_inflight_begin (struct nfs4_client *clp) +{ +atomic_inc (&clp->cl_cb_inflight); +} + +static void nfsd41_cb_inflight_end (struct nfs4_client *clp) +{ + +if (atomic_dec_and_test (&clp->cl_cb_inflight)) +wake_up_var (&clp->cl_cb_inflight); +} + +static void nfsd41_cb_inflight_wait_complete (struct nfs4_client *clp) +{ +wait_var_event (&clp->cl_cb_inflight, + !atomic_read (&clp->cl_cb_inflight)); +} +static const struct cred *get_backchannel_cred (struct nfs4_client *clp,struct rpc_clnt *client,struct nfsd4_session *ses) {if (clp->cl_minorversion ==0 ) {@@ -937,14 +962,21 @@ static void nfsd4_cb_probe_done(struct rpc_task *task, void *calldata) clp->cl_cb_state =NFSD4_CB_UP ; } +static void nfsd4_cb_probe_release (void *calldata) +{ + struct nfs4_client *clp = container_of(calldata, struct nfs4_client, cl_cb_null); + +nfsd41_cb_inflight_end (clp); + +} +static const struct rpc_call_ops nfsd4_cb_probe_ops = {/* XXX: release method to ensure we set the cb channel down if * necessary on early failure? */ .rpc_call_done = nfsd4_cb_probe_done, + .rpc_release = nfsd4_cb_probe_release, }; -static struct workqueue_struct *callback_wq; -/* * Poke the callback thread to process any updates to the callback * parameters, and send a null probe. @@ -975,9 +1007,12 @@ void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *conn)* If the slot is available, then mark it busy. Otherwise, set the * thread for sleeping on the callback RPC wait queue. */ -static bool nfsd41_cb_get_slot (struct nfs4_client *clp,struct rpc_task *task) +static bool nfsd41_cb_get_slot (struct nfsd4_callback *cb,struct rpc_task *task) { -if (test_and_set_bit (0 , &clp->cl_cb_slot_busy) !=0 ) { +struct nfs4_client *clp = cb->cb_clp; + +if (!cb->cb_holds_slot && +test_and_set_bit (0 , &clp->cl_cb_slot_busy) !=0 ) {rpc_sleep_on (&clp->cl_cb_waitq, task,NULL );/* Race breaker */ if (test_and_set_bit (0 , &clp->cl_cb_slot_busy) !=0 ) {@@ -985,10 +1020,32 @@ static bool nfsd41_cb_get_slot(struct nfs4_client *clp, struct rpc_task *task)return false; }rpc_wake_up_queued_task (&clp->cl_cb_waitq, task); + cb->cb_holds_slot = true; }return true; } +static void nfsd41_cb_release_slot (struct nfsd4_callback *cb) +{ +struct nfs4_client *clp = cb->cb_clp; + +if (cb->cb_holds_slot) { + cb->cb_holds_slot = false; +clear_bit (0 , &clp->cl_cb_slot_busy); +rpc_wake_up_next (&clp->cl_cb_waitq); + } +} + +static void nfsd41_destroy_cb (struct nfsd4_callback *cb) +{ +struct nfs4_client *clp = cb->cb_clp; + +nfsd41_cb_release_slot (cb); +if (cb->cb_ops && cb->cb_ops->release) + cb->cb_ops->release (cb); +nfsd41_cb_inflight_end (clp); +} +/* * TODO: cb_sequence should support referring call lists, cachethis, multiple * slots, and mark callback channel down on communication errors. @@ -1005,11 +1062,8 @@ static void nfsd4_cb_prepare(struct rpc_task *task, void *calldata)*/ cb->cb_seq_status =1 ; cb->cb_status =0 ; -if (minorversion) { -if (!cb->cb_holds_slot && !nfsd41_cb_get_slot (clp, task)) -return ; - cb->cb_holds_slot = true; - } +if (minorversion && !nfsd41_cb_get_slot (cb, task)) +return ;rpc_call_start (task); }@@ -1076,9 +1130,7 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback cb->cb_seq_status); } - cb->cb_holds_slot = false; -clear_bit (0 , &clp->cl_cb_slot_busy); -rpc_wake_up_next (&clp->cl_cb_waitq); +nfsd41_cb_release_slot (cb);dprintk ("%s: freed slot, new seqid=%d\n" , __func__, clp->cl_cb_session->se_cb_seq_nr);@@ -1091,8 +1143,10 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback ret = false; goto out; need_restart: - task->tk_status =0 ; - cb->cb_need_restart = true; +if (!test_bit (NFSD4_CLIENT_CB_KILL , &clp->cl_flags)) { + task->tk_status =0 ; + cb->cb_need_restart = true; + }return false; }@@ -1134,9 +1188,9 @@ static void nfsd4_cb_release(void *calldata)struct nfsd4_callback *cb = calldata;if (cb->cb_need_restart) -nfsd4_run_cb (cb); +nfsd4_queue_cb (cb);else - cb->cb_ops->release (cb); +nfsd41_destroy_cb (cb); }@@ -1170,6 +1224,7 @@ void nfsd4_shutdown_callback(struct nfs4_client *clp)*/ nfsd4_run_cb (&clp->cl_cb_null);flush_workqueue (callback_wq); +nfsd41_cb_inflight_wait_complete (clp); }/* requires cl_lock: */ @@ -1255,8 +1310,7 @@ nfsd4_run_cb_work(struct work_struct *work) clnt = clp->cl_cb_client;if (!clnt) {/* Callback channel broken, or client killed; give up: */ -if (cb->cb_ops && cb->cb_ops->release) - cb->cb_ops->release (cb); +nfsd41_destroy_cb (cb);return ; }@@ -1265,6 +1319,7 @@ nfsd4_run_cb_work(struct work_struct *work)*/ if (!cb->cb_ops && clp->cl_minorversion) { clp->cl_cb_state =NFSD4_CB_UP ; +nfsd41_destroy_cb (cb);return ; }@@ -1290,5 +1345,9 @@ void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,void nfsd4_run_cb (struct nfsd4_callback *cb) { -queue_work (callback_wq, &cb->cb_work); +struct nfs4_client *clp = cb->cb_clp; + +nfsd41_cb_inflight_begin (clp); +if (!nfsd4_queue_cb (cb)) +nfsd41_cb_inflight_end (clp); }fs/nfsd/state.h +1
-0@@ -367,6 +367,7 @@ struct nfs4_client {struct net *net;struct list_head async_copies;/* list of async copies */ spinlock_t async_lock;/* lock for async copies */ +atomic_t cl_cb_inflight;/* Outstanding callbacks */ };/* struct nfs4_client_reset Signature · 1 line
2.21.0
Quoted 11 lines · J. Bruce Fields
Quoted 4 lines · Trond Myklebust
When we're destroying the client lease, and we call
nfsd4_shutdown_callback(), we must ensure that we do not return before all outstanding callbacks have terminated and have released their payloads.This is great, thanks! We've seen what I'm fairly sure is the same bug
from Red Hat users. I think my blind spot was an assumption that rpc tasks wouldn't outliverpc_shutdown_client().
Quoted 7 lines
However, it's causing xfstests runs to hang, and I haven't worked out why yet.
I'll spend some time on it this afternoon and let you know what I figure
out.
Is that happening with v2 or with v1? With v1 there is definitely a hang in __destroy_client() due to the refcount leak that I believe I fixed in v2.
Quoted 238 lines
--b.
Quoted 236 lines
Signed-off-by Trond Myklebust <[email protected]>---
v2 - Fix a leak ofclp->cl_cb_inflightwhen running null probesfs/nfsd/nfs4callback.c | 97 +++++++++++++++++++++++++++++++++----- ---- fs/nfsd/state.h | 1 + 2 files changed, 79 insertions(+), 19 deletions(-)fs/nfsd/nfs4callback.c +25
-0@@ -826,6 +826,31 @@ static int max_cb_time(struct net *net)return max (nn->nfsd4_lease/10 , (time_t )1 ) *HZ ; } +static struct workqueue_struct *callback_wq; + +static bool nfsd4_queue_cb (struct nfsd4_callback *cb) +{ +return queue_work (callback_wq, &cb->cb_work); +} + +static void nfsd41_cb_inflight_begin (struct nfs4_client *clp) +{ +atomic_inc (&clp->cl_cb_inflight); +} + +static void nfsd41_cb_inflight_end (struct nfs4_client *clp) +{ + +if (atomic_dec_and_test (&clp->cl_cb_inflight)) +wake_up_var (&clp->cl_cb_inflight); +} + +static void nfsd41_cb_inflight_wait_complete (struct nfs4_client *clp) +{ +wait_var_event (&clp->cl_cb_inflight, + !atomic_read (&clp->cl_cb_inflight)); +} +static const struct cred *get_backchannel_cred (struct nfs4_client *clp,struct rpc_clnt *client,struct nfsd4_session *ses){
fs/nfsd/nfs4callback.c +9
-2if (clp->cl_minorversion ==0 ) {@@ -937,14 +962,21 @@ static void nfsd4_cb_probe_done(struct rpc_task *task, void *calldata) clp->cl_cb_state =NFSD4_CB_UP ; } +static void nfsd4_cb_probe_release (void *calldata) +{ + struct nfs4_client *clp = container_of(calldata, struct nfs4_client, cl_cb_null); + +nfsd41_cb_inflight_end (clp); + +} +static const struct rpc_call_ops nfsd4_cb_probe_ops = {/* XXX: release method to ensure we set the cb channel down if * necessary on early failure? */ .rpc_call_done = nfsd4_cb_probe_done, + .rpc_release = nfsd4_cb_probe_release, }; -static struct workqueue_struct *callback_wq; -/* * Poke the callback thread to process any updates to the callback
* parameters, and send a null probe.fs/nfsd/nfs4callback.c +0
-0@@ -975,9 +1007,12 @@ void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *conn) * If the slot is available, then mark it busy. Otherwise, setthe
* thread for sleeping on the callback RPC wait queue. */
-static boolnfsd41_cb_get_slot(struct nfs4_client *clp, struct rpc_task *task)
+static boolnfsd41_cb_get_slot(struct nfsd4_callback *cb, struct rpc_task *task)
{fs/nfsd/nfs4callback.c +26
-1-if (test_and_set_bit (0 , &clp->cl_cb_slot_busy) !=0 ) { +struct nfs4_client *clp = cb->cb_clp; + +if (!cb->cb_holds_slot && +test_and_set_bit (0 , &clp->cl_cb_slot_busy) !=0 ) {rpc_sleep_on (&clp->cl_cb_waitq, task,NULL );/* Race breaker */ if (test_and_set_bit (0 , &clp->cl_cb_slot_busy) !=0 ) {@@ -985,10 +1020,32 @@ static bool nfsd41_cb_get_slot(struct nfs4_client *clp, struct rpc_task *task) return false; }rpc_wake_up_queued_task (&clp->cl_cb_waitq, task); + cb->cb_holds_slot = true; }return true; } +static void nfsd41_cb_release_slot (struct nfsd4_callback *cb) +{ +struct nfs4_client *clp = cb->cb_clp; + +if (cb->cb_holds_slot) { + cb->cb_holds_slot = false; +clear_bit (0 , &clp->cl_cb_slot_busy); +rpc_wake_up_next (&clp->cl_cb_waitq); + } +} + +static void nfsd41_destroy_cb (struct nfsd4_callback *cb) +{ +struct nfs4_client *clp = cb->cb_clp; + +nfsd41_cb_release_slot (cb); +if (cb->cb_ops && cb->cb_ops->release) + cb->cb_ops->release (cb); +nfsd41_cb_inflight_end (clp); +} +/* * TODO: cb_sequence should support referring call lists, cachethis, multiple
* slots, and mark callback channel down on communication errors.fs/nfsd/nfs4callback.c +2
-5@@ -1005,11 +1062,8 @@ static void nfsd4_cb_prepare(struct rpc_task*task, void *calldata) */ cb->cb_seq_status =1 ; cb->cb_status =0 ; -if (minorversion) { -if (!cb->cb_holds_slot && !nfsd41_cb_get_slot (clp, task)) -return ; - cb->cb_holds_slot = true; - } +if (minorversion && !nfsd41_cb_get_slot (cb, task)) +return ;rpc_call_start (task);}
fs/nfsd/nfs4callback.c +0
-0@@ -1076,9 +1130,7 @@ static bool nfsd4_cb_sequence_done(struct
rpc_task*task, structnfsd4_callbackfs/nfsd/nfs4callback.c +1
-3cb->cb_seq_status); } - cb->cb_holds_slot = false; -clear_bit (0 , &clp->cl_cb_slot_busy); -rpc_wake_up_next (&clp->cl_cb_waitq); +nfsd41_cb_release_slot (cb);dprintk ("%s: freed slot, new seqid=%d\n" , __func__, clp->cl_cb_session->se_cb_seq_nr);@@ -1091,8 +1143,10 @@ static bool nfsd4_cb_sequence_done(struct
rpc_task*task, structnfsd4_callbackfs/nfsd/nfs4callback.c +7
-4ret = false; goto out; need_restart: - task->tk_status =0 ; - cb->cb_need_restart = true; +if (!test_bit (NFSD4_CLIENT_CB_KILL , &clp->cl_flags)) { + task->tk_status =0 ; + cb->cb_need_restart = true; + }return false; }@@ -1134,9 +1188,9 @@ static void nfsd4_cb_release(void *calldata)struct nfsd4_callback *cb = calldata;if (cb->cb_need_restart) -nfsd4_run_cb (cb); +nfsd4_queue_cb (cb);else - cb->cb_ops->release (cb); +nfsd41_destroy_cb (cb); }@@ -1170,6 +1224,7 @@ void nfsd4_shutdown_callback(structnfs4_client *clp) */ nfsd4_run_cb (&clp->cl_cb_null);flush_workqueue (callback_wq); +nfsd41_cb_inflight_wait_complete (clp); }/* requires
cl_lock: */fs/nfsd/nfs4callback.c +7
-3@@ -1255,8 +1310,7 @@ nfsd4_run_cb_work(struct work_struct *work) clnt = clp->cl_cb_client;if (!clnt) {/* Callback channel broken, or client killed; give up: */ -if (cb->cb_ops && cb->cb_ops->release) - cb->cb_ops->release (cb); +nfsd41_destroy_cb (cb);return ; }@@ -1265,6 +1319,7 @@ nfsd4_run_cb_work(struct work_struct *work)*/ if (!cb->cb_ops && clp->cl_minorversion) { clp->cl_cb_state =NFSD4_CB_UP ; +nfsd41_destroy_cb (cb);return ; }@@ -1290,5 +1345,9 @@ void nfsd4_init_cb(struct nfsd4_callback *cb,struct nfs4_client *clp, void nfsd4_run_cb(struct nfsd4_callback *cb) { - queue_work(callback_wq, &cb->cb_work); + struct nfs4_client *clp = cb->cb_clp; + +nfsd41_cb_inflight_begin (clp); +if (!nfsd4_queue_cb (cb)) +nfsd41_cb_inflight_end (clp);}
fs/nfsd/state.h +1
-0@@ -367,6 +367,7 @@ struct nfs4_client {struct net *net;struct list_head async_copies;/* list of async copies */ spinlock_t async_lock;/* lock for async copies */ +atomic_t cl_cb_inflight;/* Outstanding callbacks */ };/* struct nfs4_client_reset Signature · 1 line
2.21.0
Signature · 3 lines
Trond Myklebust Linux NFS client maintainer, Hammerspace [email protected]
Quoted 15 lines · Trond Myklebust
Quoted 14 lines · J. Bruce Fields
Quoted 4 lines · Trond Myklebust
When we're destroying the client lease, and we call
nfsd4_shutdown_callback(), we must ensure that we do not return before all outstanding callbacks have terminated and have released their payloads.This is great, thanks! We've seen what I'm fairly sure is the same bug
from Red Hat users. I think my blind spot was an assumption that rpc tasks wouldn't outliverpc_shutdown_client().However, it's causing xfstests runs to hang, and I haven't worked out why yet.
Quoted 8 lines
Quoted 4 lines
I'll spend some time on it this afternoon and let you know what I figure
out.Is that happening with v2 or with v1? With v1 there is definitely a hang in
__destroy_client()due to the refcount leak that I believe I fixed in v2.
I thought I was running v2, let me double-check....
--b.
Quoted 245 lines
Quoted 239 lines
--b.
Quoted 236 lines
Signed-off-by Trond Myklebust <[email protected]>---
v2 - Fix a leak ofclp->cl_cb_inflightwhen running null probesfs/nfsd/nfs4callback.c | 97 +++++++++++++++++++++++++++++++++----- ---- fs/nfsd/state.h | 1 + 2 files changed, 79 insertions(+), 19 deletions(-)fs/nfsd/nfs4callback.c +25
-0@@ -826,6 +826,31 @@ static int max_cb_time(struct net *net)return max (nn->nfsd4_lease/10 , (time_t )1 ) *HZ ; } +static struct workqueue_struct *callback_wq; + +static bool nfsd4_queue_cb (struct nfsd4_callback *cb) +{ +return queue_work (callback_wq, &cb->cb_work); +} + +static void nfsd41_cb_inflight_begin (struct nfs4_client *clp) +{ +atomic_inc (&clp->cl_cb_inflight); +} + +static void nfsd41_cb_inflight_end (struct nfs4_client *clp) +{ + +if (atomic_dec_and_test (&clp->cl_cb_inflight)) +wake_up_var (&clp->cl_cb_inflight); +} + +static void nfsd41_cb_inflight_wait_complete (struct nfs4_client *clp) +{ +wait_var_event (&clp->cl_cb_inflight, + !atomic_read (&clp->cl_cb_inflight)); +} +static const struct cred *get_backchannel_cred (struct nfs4_client *clp,struct rpc_clnt *client,struct nfsd4_session *ses){
fs/nfsd/nfs4callback.c +9
-2if (clp->cl_minorversion ==0 ) {@@ -937,14 +962,21 @@ static void nfsd4_cb_probe_done(struct rpc_task *task, void *calldata) clp->cl_cb_state =NFSD4_CB_UP ; } +static void nfsd4_cb_probe_release (void *calldata) +{ + struct nfs4_client *clp = container_of(calldata, struct nfs4_client, cl_cb_null); + +nfsd41_cb_inflight_end (clp); + +} +static const struct rpc_call_ops nfsd4_cb_probe_ops = {/* XXX: release method to ensure we set the cb channel down if * necessary on early failure? */ .rpc_call_done = nfsd4_cb_probe_done, + .rpc_release = nfsd4_cb_probe_release, }; -static struct workqueue_struct *callback_wq; -/* * Poke the callback thread to process any updates to the callback
* parameters, and send a null probe.fs/nfsd/nfs4callback.c +0
-0@@ -975,9 +1007,12 @@ void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *conn) * If the slot is available, then mark it busy. Otherwise, setthe
* thread for sleeping on the callback RPC wait queue. */
-static boolnfsd41_cb_get_slot(struct nfs4_client *clp, struct rpc_task *task)
+static boolnfsd41_cb_get_slot(struct nfsd4_callback *cb, struct rpc_task *task)
{fs/nfsd/nfs4callback.c +26
-1-if (test_and_set_bit (0 , &clp->cl_cb_slot_busy) !=0 ) { +struct nfs4_client *clp = cb->cb_clp; + +if (!cb->cb_holds_slot && +test_and_set_bit (0 , &clp->cl_cb_slot_busy) !=0 ) {rpc_sleep_on (&clp->cl_cb_waitq, task,NULL );/* Race breaker */ if (test_and_set_bit (0 , &clp->cl_cb_slot_busy) !=0 ) {@@ -985,10 +1020,32 @@ static bool nfsd41_cb_get_slot(struct nfs4_client *clp, struct rpc_task *task) return false; }rpc_wake_up_queued_task (&clp->cl_cb_waitq, task); + cb->cb_holds_slot = true; }return true; } +static void nfsd41_cb_release_slot (struct nfsd4_callback *cb) +{ +struct nfs4_client *clp = cb->cb_clp; + +if (cb->cb_holds_slot) { + cb->cb_holds_slot = false; +clear_bit (0 , &clp->cl_cb_slot_busy); +rpc_wake_up_next (&clp->cl_cb_waitq); + } +} + +static void nfsd41_destroy_cb (struct nfsd4_callback *cb) +{ +struct nfs4_client *clp = cb->cb_clp; + +nfsd41_cb_release_slot (cb); +if (cb->cb_ops && cb->cb_ops->release) + cb->cb_ops->release (cb); +nfsd41_cb_inflight_end (clp); +} +/* * TODO: cb_sequence should support referring call lists, cachethis, multiple
* slots, and mark callback channel down on communication errors.fs/nfsd/nfs4callback.c +2
-5@@ -1005,11 +1062,8 @@ static void nfsd4_cb_prepare(struct rpc_task*task, void *calldata) */ cb->cb_seq_status =1 ; cb->cb_status =0 ; -if (minorversion) { -if (!cb->cb_holds_slot && !nfsd41_cb_get_slot (clp, task)) -return ; - cb->cb_holds_slot = true; - } +if (minorversion && !nfsd41_cb_get_slot (cb, task)) +return ;rpc_call_start (task);}
fs/nfsd/nfs4callback.c +0
-0@@ -1076,9 +1130,7 @@ static bool nfsd4_cb_sequence_done(struct
rpc_task*task, structnfsd4_callbackfs/nfsd/nfs4callback.c +1
-3cb->cb_seq_status); } - cb->cb_holds_slot = false; -clear_bit (0 , &clp->cl_cb_slot_busy); -rpc_wake_up_next (&clp->cl_cb_waitq); +nfsd41_cb_release_slot (cb);dprintk ("%s: freed slot, new seqid=%d\n" , __func__, clp->cl_cb_session->se_cb_seq_nr);@@ -1091,8 +1143,10 @@ static bool nfsd4_cb_sequence_done(struct
rpc_task*task, structnfsd4_callbackfs/nfsd/nfs4callback.c +7
-4ret = false; goto out; need_restart: - task->tk_status =0 ; - cb->cb_need_restart = true; +if (!test_bit (NFSD4_CLIENT_CB_KILL , &clp->cl_flags)) { + task->tk_status =0 ; + cb->cb_need_restart = true; + }return false; }@@ -1134,9 +1188,9 @@ static void nfsd4_cb_release(void *calldata)struct nfsd4_callback *cb = calldata;if (cb->cb_need_restart) -nfsd4_run_cb (cb); +nfsd4_queue_cb (cb);else - cb->cb_ops->release (cb); +nfsd41_destroy_cb (cb); }@@ -1170,6 +1224,7 @@ void nfsd4_shutdown_callback(structnfs4_client *clp) */ nfsd4_run_cb (&clp->cl_cb_null);flush_workqueue (callback_wq); +nfsd41_cb_inflight_wait_complete (clp); }/* requires
cl_lock: */fs/nfsd/nfs4callback.c +7
-3@@ -1255,8 +1310,7 @@ nfsd4_run_cb_work(struct work_struct *work) clnt = clp->cl_cb_client;if (!clnt) {/* Callback channel broken, or client killed; give up: */ -if (cb->cb_ops && cb->cb_ops->release) - cb->cb_ops->release (cb); +nfsd41_destroy_cb (cb);return ; }@@ -1265,6 +1319,7 @@ nfsd4_run_cb_work(struct work_struct *work)*/ if (!cb->cb_ops && clp->cl_minorversion) { clp->cl_cb_state =NFSD4_CB_UP ; +nfsd41_destroy_cb (cb);return ; }@@ -1290,5 +1345,9 @@ void nfsd4_init_cb(struct nfsd4_callback *cb,struct nfs4_client *clp, void nfsd4_run_cb(struct nfsd4_callback *cb) { - queue_work(callback_wq, &cb->cb_work); + struct nfs4_client *clp = cb->cb_clp; + +nfsd41_cb_inflight_begin (clp); +if (!nfsd4_queue_cb (cb)) +nfsd41_cb_inflight_end (clp);}
fs/nfsd/state.h +1
-0@@ -367,6 +367,7 @@ struct nfs4_client {struct net *net;struct list_head async_copies;/* list of async copies */ spinlock_t async_lock;/* lock for async copies */ +atomic_t cl_cb_inflight;/* Outstanding callbacks */ };/* struct nfs4_client_reset Signature · 1 line
2.21.0Signature · 3 lines
Trond Myklebust Linux NFS client maintainer, Hammerspace [email protected]
Quoted 21 lines · J. Bruce Fields
Quoted 20 lines · Trond Myklebust
Quoted 18 lines · J. Bruce Fields
Quoted 4 lines · Trond Myklebust
When we're destroying the client lease, and we call
nfsd4_shutdown_callback(), we must ensure that we do not return before all outstanding callbacks have terminated and have released their payloads.This is great, thanks! We've seen what I'm fairly sure is the same bug
from Red Hat users. I think my blind spot was an assumption that rpc tasks wouldn't outliverpc_shutdown_client().However, it's causing xfstests runs to hang, and I haven't worked out why yet.
I'll spend some time on it this afternoon and let you know what I figure
out.
Quoted 5 lines
Quoted 3 lines
Is that happening with v2 or with v1? With v1 there is definitely a hang in
__destroy_client()due to the refcount leak that I believe I fixed in v2.I thought I was running v2, let me double-check....
Yes, with v2 I'm getting a hang on generic/013.
I checked quickly and didn't see anything interesting in the logs, otherwise I haven't done any digging.
--b.
Quoted 7 lines · bfields
Quoted 1 line · J. Bruce Fields
I thought I was running v2, let me double-check....
Yes, with v2 I'm getting a hang on generic/013.
I checked quickly and didn't see anything interesting in the logs, otherwise I haven't done any digging.
Reproduceable just with ./check -nfs generic/013. The last thing I see in wireshark is an asynchronous COPY call and reply. Which means it's probably trying to do a CB_OFFLOAD. Hm.
--b.
Quoted 4 lines · J. Bruce Fields
Quoted 3 lines · bfields
Quoted 1 line · J. Bruce Fields
I thought I was running v2, let me double-check....
Quoted 8 lines
Quoted 4 lines
Yes, with v2 I'm getting a hang on generic/013.
I checked quickly and didn't see anything interesting in the logs, otherwise I haven't done any digging.
Reproduceable just with ./check -nfs generic/013. The last thing I see in wireshark is an asynchronous COPY call and reply. Which means it's probably trying to do a
CB_OFFLOAD. Hm.
Oh, I think it just needs the following.
--b.
fs/nfsd/nfs4callback.c +1 -1
@@ -1026,8 +1026,8 @@ static bool nfsd41_cb_get_slot(struct nfsd4_callback *cb, struct rpc_task *task)return false; }rpc_wake_up_queued_task (&clp->cl_cb_waitq, task); - cb->cb_holds_slot = true; } + cb->cb_holds_slot = true;return true; }
Quoted 7 lines · J. Bruce Fields
Quoted 6 lines · J. Bruce Fields
Quoted 5 lines · bfields
Quoted 1 line · J. Bruce Fields
I thought I was running v2, let me double-check....
Yes, with v2 I'm getting a hang on generic/013.
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
I checked quickly and didn't see anything interesting in the logs, otherwise I haven't done any digging.
Reproduceable just with ./check -nfs generic/013. The last thing I see in wireshark is an asynchronous COPY call and reply. Which means it's probably trying to do a
CB_OFFLOAD. Hm.Oh, I think it just needs the following.
Applying as follows, with part of the change split out into a separate patch (since that's how I noticed the bug).
--b.
Quoted 15 lines · J. Bruce Fields
Quoted 14 lines · J. Bruce Fields
Quoted 12 lines · J. Bruce Fields
Quoted 7 lines · bfields
Quoted 1 line · J. Bruce Fields
I thought I was running v2, let me double-check....
Yes, with v2 I'm getting a hang on generic/013.
I checked quickly and didn't see anything interesting in the logs, otherwise I haven't done any digging.
Reproduceable just with ./check -nfs generic/013. The last thing I see in wireshark is an asynchronous COPY call and reply. Which means it's probably trying to do a
CB_OFFLOAD. Hm.
Quoted 6 lines
Quoted 1 line
Oh, I think it just needs the following.
Applying as follows, with part of the change split out into a separate patch (since that's how I noticed the bug).
--b.
Oops, remembered to append the patches this time.--b.
commit 2bbfed98a4d8
Author: Trond Myklebust <[email protected]> Date: Wed Oct 23 17:43:18 2019 -0400
nfsd: Fix races between nfsd4_cb_release() and nfsd4_shutdown_callback()
When we're destroying the client lease, and we call nfsd4_shutdown_callback(), we must ensure that we do not return before all outstanding callbacks have terminated and have released their payloads.
fs/nfsd/nfs4callback.c +58 -9
@@ -826,6 +826,31 @@ static int max_cb_time(struct net *net)return max (nn->nfsd4_lease/10 , (time_t )1 ) *HZ ; } +static struct workqueue_struct *callback_wq; + +static bool nfsd4_queue_cb (struct nfsd4_callback *cb) +{ +return queue_work (callback_wq, &cb->cb_work); +} + +static void nfsd41_cb_inflight_begin (struct nfs4_client *clp) +{ +atomic_inc (&clp->cl_cb_inflight); +} + +static void nfsd41_cb_inflight_end (struct nfs4_client *clp) +{ + +if (atomic_dec_and_test (&clp->cl_cb_inflight)) +wake_up_var (&clp->cl_cb_inflight); +} + +static void nfsd41_cb_inflight_wait_complete (struct nfs4_client *clp) +{ +wait_var_event (&clp->cl_cb_inflight, + !atomic_read (&clp->cl_cb_inflight)); +} +static const struct cred *get_backchannel_cred (struct nfs4_client *clp,struct rpc_clnt *client,struct nfsd4_session *ses) {if (clp->cl_minorversion ==0 ) {
@@ -937,14 +962,21 @@ static void nfsd4_cb_probe_done(struct rpc_task *task, void *calldata) clp->cl_cb_state =NFSD4_CB_UP ; } +static void nfsd4_cb_probe_release (void *calldata) +{ + struct nfs4_client *clp = container_of(calldata, struct nfs4_client, cl_cb_null); + +nfsd41_cb_inflight_end (clp); + +} +static const struct rpc_call_ops nfsd4_cb_probe_ops = {/* XXX: release method to ensure we set the cb channel down if * necessary on early failure? */ .rpc_call_done = nfsd4_cb_probe_done, + .rpc_release = nfsd4_cb_probe_release, }; -static struct workqueue_struct *callback_wq; -/* * Poke the callback thread to process any updates to the callback * parameters, and send a null probe.
@@ -1004,6 +1036,16 @@ static void nfsd41_cb_release_slot(struct nfsd4_callback *cb) } } +static void nfsd41_destroy_cb (struct nfsd4_callback *cb) +{ +struct nfs4_client *clp = cb->cb_clp; + +nfsd41_cb_release_slot (cb); +if (cb->cb_ops && cb->cb_ops->release) + cb->cb_ops->release (cb); +nfsd41_cb_inflight_end (clp); +} +/* * TODO: cb_sequence should support referring call lists, cachethis, multiple * slots, and mark callback channel down on communication errors.
@@ -1101,8 +1143,10 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback ret = false; goto out; need_restart: - task->tk_status =0 ; - cb->cb_need_restart = true; +if (!test_bit (NFSD4_CLIENT_CB_KILL , &clp->cl_flags)) { + task->tk_status =0 ; + cb->cb_need_restart = true; + }return false; }
@@ -1144,9 +1188,9 @@ static void nfsd4_cb_release(void *calldata)struct nfsd4_callback *cb = calldata;if (cb->cb_need_restart) -nfsd4_run_cb (cb); +nfsd4_queue_cb (cb);else - cb->cb_ops->release (cb); +nfsd41_destroy_cb (cb); }
@@ -1180,6 +1224,7 @@ void nfsd4_shutdown_callback(struct nfs4_client *clp)*/ nfsd4_run_cb (&clp->cl_cb_null);flush_workqueue (callback_wq); +nfsd41_cb_inflight_wait_complete (clp); }/* requires cl_lock: */
@@ -1265,8 +1310,7 @@ nfsd4_run_cb_work(struct work_struct *work) clnt = clp->cl_cb_client;if (!clnt) {/* Callback channel broken, or client killed; give up: */ -if (cb->cb_ops && cb->cb_ops->release) - cb->cb_ops->release (cb); +nfsd41_destroy_cb (cb);return ; }
@@ -1275,6 +1319,7 @@ nfsd4_run_cb_work(struct work_struct *work)*/ if (!cb->cb_ops && clp->cl_minorversion) { clp->cl_cb_state =NFSD4_CB_UP ; +nfsd41_destroy_cb (cb);return ; }
@@ -1300,5 +1345,9 @@ void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,void nfsd4_run_cb (struct nfsd4_callback *cb) { -queue_work (callback_wq, &cb->cb_work); +struct nfs4_client *clp = cb->cb_clp; + +nfsd41_cb_inflight_begin (clp); +if (!nfsd4_queue_cb (cb)) +nfsd41_cb_inflight_end (clp); }
fs/nfsd/state.h +1 -0
@@ -367,6 +367,7 @@ struct nfs4_client {struct net *net;struct list_head async_copies;/* list of async copies */ spinlock_t async_lock;/* lock for async copies */ +atomic_t cl_cb_inflight;/* Outstanding callbacks */ };/* struct nfs4_client_reset
commit 12357f1b2c8e
Author: Trond Myklebust <[email protected]> Date: Thu Nov 7 17:11:57 2019 -0500
nfsd: minor 4.1 callback cleanup
Move all the cb_holds_slot management into helper functions. No change in behavior.
fs/nfsd/nfs4callback.c +20 -10
@@ -975,9 +975,12 @@ void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *conn)* If the slot is available, then mark it busy. Otherwise, set the * thread for sleeping on the callback RPC wait queue. */ -static bool nfsd41_cb_get_slot (struct nfs4_client *clp,struct rpc_task *task) +static bool nfsd41_cb_get_slot (struct nfsd4_callback *cb,struct rpc_task *task) { -if (test_and_set_bit (0 , &clp->cl_cb_slot_busy) !=0 ) { +struct nfs4_client *clp = cb->cb_clp; + +if (!cb->cb_holds_slot && +test_and_set_bit (0 , &clp->cl_cb_slot_busy) !=0 ) {rpc_sleep_on (&clp->cl_cb_waitq, task,NULL );/* Race breaker */ if (test_and_set_bit (0 , &clp->cl_cb_slot_busy) !=0 ) {
@@ -986,9 +989,21 @@ static bool nfsd41_cb_get_slot(struct nfs4_client *clp, struct rpc_task *task) }rpc_wake_up_queued_task (&clp->cl_cb_waitq, task); } + cb->cb_holds_slot = true;return true; } +static void nfsd41_cb_release_slot (struct nfsd4_callback *cb) +{ +struct nfs4_client *clp = cb->cb_clp; + +if (cb->cb_holds_slot) { + cb->cb_holds_slot = false; +clear_bit (0 , &clp->cl_cb_slot_busy); +rpc_wake_up_next (&clp->cl_cb_waitq); + } +} +/* * TODO: cb_sequence should support referring call lists, cachethis, multiple * slots, and mark callback channel down on communication errors.
@@ -1005,11 +1020,8 @@ static void nfsd4_cb_prepare(struct rpc_task *task, void *calldata)*/ cb->cb_seq_status =1 ; cb->cb_status =0 ; -if (minorversion) { -if (!cb->cb_holds_slot && !nfsd41_cb_get_slot (clp, task)) -return ; - cb->cb_holds_slot = true; - } +if (minorversion && !nfsd41_cb_get_slot (cb, task)) +return ;rpc_call_start (task); }
@@ -1076,9 +1088,7 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback cb->cb_seq_status); } - cb->cb_holds_slot = false; -clear_bit (0 , &clp->cl_cb_slot_busy); -rpc_wake_up_next (&clp->cl_cb_waitq); +nfsd41_cb_release_slot (cb);dprintk ("%s: freed slot, new seqid=%d\n" , __func__, clp->cl_cb_session->se_cb_seq_nr);
PATCH 1/1 nfsd: document callback_wq serialization of callback code
From: "J. Bruce Fields" <[email protected]>
Date: Tue, 29 Oct 2019 16:02:18 -0400
The callback code relies on the fact that much of it is only ever called from the ordered workqueue callback_wq, and this is worth documenting.
--- fs/nfsd/nfs4callback.c | 6 ++++++ 1 file changed, 6 insertions(+)
Also adding a comment, since I know this was a source of confusion when investigating these races.
fs/nfsd/nfs4callback.c +6 -0
@@ -1243,6 +1243,12 @@ static struct nfsd4_conn * __nfsd4_find_backchannel(struct nfs4_client *clp)return NULL ; } +/* +* Note there isn't a lot of locking in this code; instead we depend on +* the fact that it is run from the callback_wq, which won't run two +* work items at once. So, for example, callback_wq handles all access +* of cl_cb_client and all calls to rpc_create or rpc_shutdown_client. +*/ static void nfsd4_process_cb_update (struct nfsd4_callback *cb) {struct nfs4_cb_conn conn;
Signature · 1 line
2.23.0
PATCH 1/1 nfsd: mark cb path down on unknown errors
From: "J. Bruce Fields" <[email protected]>
Date: Tue, 22 Oct 2019 12:29:37 -0400
An unexpected error is probably a sign that something is wrong with the callback path.
--- fs/nfsd/nfs4callback.c | 1 + 1 file changed, 1 insertion(+)
Also, while we're here....--b.
fs/nfsd/nfs4callback.c +1 -0
@@ -1126,6 +1126,7 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback }break ;default : +nfsd4_mark_cb_fault (cb->cb_clp, cb->cb_seq_status);dprintk ("%s: unprocessed error %d\n" , __func__, cb->cb_seq_status); }
Signature · 1 line
2.23.0
Quoted 27 lines · J. Bruce Fields
Quoted 10 lines · J. Bruce Fields
Quoted 8 lines · bfields
Quoted 1 line · J. Bruce Fields
I thought I was running v2, let me double-check....
Yes, with v2 I'm getting a hang on generic/013.
I checked quickly and didn't see anything interesting in the logs,
otherwise I haven't done any digging.Quoted 5 lines
Reproduceable just with ./check -nfs generic/013. The last thing I see
in wireshark is an asynchronous COPY call and reply. Which means it's
probably trying to do aCB_OFFLOAD. Hm.Oh, I think it just needs the following.
--b.
fs/nfsd/nfs4callback.c +0
-0
fs/nfsd/nfs4callback.c static bool nfsd41_cb_get_slot(struct · line 1033
Quoted 9 lines
fs/nfsd/nfs4callback.c +1
-1@@ -1026,8 +1026,8 @@ static bool nfsd41_cb_get_slot(struct nfsd4_callback *cb, struct rpc_task *task) return false; }rpc_wake_up_queued_task (&clp->cl_cb_waitq, task); - cb->cb_holds_slot = true; } + cb->cb_holds_slot = true;return true; }
Doh! Thanks for spotting that.
Signature · 3 lines
Trond Myklebust Linux NFS client maintainer, Hammerspace [email protected]