nfs41 Fix session reset deadlocks
Subsystems (2)
cover nfs41 Fix session reset deadlocks
Fix session reset deadlocks. These patches apply to tronds nfs-2.6 bugfix branch (2.6.32-rc6 on top of these two already posted patches:
nfs41-fix-multiple-free-slot-calls.patch nfs41-fix-race-on-session-reset.patch
Fix two problems reported by Trond:
0001-nfs41-remove-nfs4_recover_session.patch 0002-nfs41-fix-state-manager-deadlock-in-session-reset.patch
Testing:
CONFIG_NFS_V4_1
v41 mount: Connectathon tests passed. PyNFS testclient.py SESSIONRESET tests and the new INJECT_ERROR_CTHON test where NFS4ERR_BADSESSION was returned every 50th SEQUENCE operation durring a Connectathon basic test run. This passed with the session being reset 390 times from a variety of client compounds (READ/WRITE/COMMIT/OPEN/CLOSE/ to name a few).
v4 mount: Connectathon tests passed.
no CONFIG_NFS_V4_1
v4 mount: Connectathon tests passed.
-->Andy
PATCH 1/2 nfs41: remove nfs4_recover_session
From: Andy Adamson <[email protected]>
nfs4_recover_session can put rpciod to sleep. Just use nfs4_schedule_recovery.
--- fs/nfs/nfs4proc.c | 26 +++----------------------- 1 files changed, 3 insertions(+), 23 deletions(-)
fs/nfs/nfs4proc.c +3 -23
@@ -427,24 +427,6 @@ out:return ret_id; } -static int nfs4_recover_session (struct nfs4_session *session) -{ -struct nfs_client *clp = session->clp; -unsigned int loop; -int ret; - -for (loop =NFS4_MAX_LOOP_ON_RECOVER ; loop !=0 ; loop--) { - ret =nfs4_wait_clnt_recover (clp); -if (ret !=0 ) -break ; -if (!test_bit (NFS4CLNT_SESSION_SETUP , &clp->cl_state)) -break ; -nfs4_schedule_state_manager (clp); - ret = -EIO ; - } -return ret; -} - static int nfs41_setup_sequence(struct nfs4_session *session, struct nfs4_sequence_args *args, struct nfs4_sequence_res *res,
@@ -453,7 +435,6 @@ static int nfs41_setup_sequence(struct nfs4_session *session, {struct nfs4_slot *slot;struct nfs4_slot_table *tbl; -int status =0 ;u8 slotid;dprintk ("--> %s\n" , __func__);
@@ -476,11 +457,10 @@ static int nfs41_setup_sequence(struct nfs4_session *session,/* The slot table is empty; start the reset thread */ dprintk ("%s Session Reset\n" , __func__); +rpc_sleep_on (&tbl->slot_tbl_waitq, task,NULL ); +nfs4_schedule_state_manager (session->clp);spin_unlock (&tbl->slot_tbl_lock); - status =nfs4_recover_session (session); -if (status) -return status; -spin_lock (&tbl->slot_tbl_lock); +return -EAGAIN ; } slotid =nfs4_find_slot (tbl, task);
Signature · 1 line
1.6.0.6
PATCH 2/2 nfs41: fix state manager deadlock in session reset
From: Andy Adamson <[email protected]>
If the session is reset during state recovery, the state manager thread can sleep on the slot_tbl_waitq causing a deadlock.
Add a completion framework to the session. Have the state manager thread set a new session state (NFS4CLNT_SESSION_DRAINING) and wait for the session slot table to drain.
Signal the state manager thread in nfs41_sequence_free_slot when the NFS4CLNT_SESSION_DRAINING bit is set and the session is drained.
--- fs/nfs/nfs4_fs.h | 1 + fs/nfs/nfs4proc.c | 25 ++++++++++++++++--------- fs/nfs/nfs4state.c | 15 +++++++++++++++ include/linux/nfs_fs_sb.h | 1 + 4 files changed, 33 insertions(+), 9 deletions(-)
fs/nfs/nfs4_fs.h +1 -0
@@ -45,6 +45,7 @@ enum nfs4_client_state {NFS4CLNT_RECLAIM_NOGRACE ,NFS4CLNT_DELEGRETURN ,NFS4CLNT_SESSION_SETUP , + NFS4CLNT_SESSION_DRAINING, };/*
fs/nfs/nfs4proc.c +16 -9
@@ -355,6 +355,16 @@ void nfs41_sequence_free_slot(const struct nfs_client *clp, }nfs4_free_slot (tbl, res->sr_slotid); res->sr_slotid =NFS4_MAX_SLOT_TABLE ; + +/* Signal state manager thread if session is drained */ +if (test_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state)) { +spin_lock (&tbl->slot_tbl_lock); +if (tbl->highest_used_slotid ==-1 ) { +dprintk ("%s COMPLETE: Session Drained\n" , __func__); +complete (&clp->cl_session->complete); + } +spin_unlock (&tbl->slot_tbl_lock); + } } static void nfs41_sequence_done(struct nfs_client *clp,
@@ -448,15 +458,11 @@ static int nfs41_setup_sequence(struct nfs4_session *session,spin_lock (&tbl->slot_tbl_lock);if (test_bit (NFS4CLNT_SESSION_SETUP , &session->clp->cl_state)) { -if (tbl->highest_used_slotid !=-1 ) { -rpc_sleep_on (&tbl->slot_tbl_waitq, task,NULL ); -spin_unlock (&tbl->slot_tbl_lock); -dprintk ("<-- %s: Session reset: draining\n" , __func__); -return -EAGAIN ; - } - -/* The slot table is empty; start the reset thread */ -dprintk ("%s Session Reset\n" , __func__); +/* +* The state manager will wait until the slot table is empty. +* Schedule the reset thread +*/ +dprintk ("%s Schedule Session Reset\n" , __func__);rpc_sleep_on (&tbl->slot_tbl_waitq, task,NULL );nfs4_schedule_state_manager (session->clp);spin_unlock (&tbl->slot_tbl_lock);
@@ -4583,6 +4589,7 @@ struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp)* nfs_client struct */ clp->cl_cons_state =NFS_CS_SESSION_INITING ; +init_completion (&session->complete); tbl = &session->fc_slot_table;spin_lock_init (&tbl->slot_tbl_lock);
fs/nfs/nfs4state.c +15 -0
@@ -1162,8 +1162,23 @@ static void nfs4_session_recovery_handle_error(struct nfs_client *clp, int err)static int nfs4_reset_session (struct nfs_client *clp) { +struct nfs4_session *ses = clp->cl_session; +struct nfs4_slot_table *tbl = &ses->fc_slot_table;int status; +spin_lock (&tbl->slot_tbl_lock); +if (tbl->highest_used_slotid !=-1 ) { +set_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state); +spin_unlock (&tbl->slot_tbl_lock); + status =wait_for_completion_interruptible (&ses->complete); +clear_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state); +if (status)/* -ERESTARTSYS */ + goto out; +INIT_COMPLETION (ses->complete); + }else { +spin_unlock (&tbl->slot_tbl_lock); + } + status =nfs4_proc_destroy_session (clp->cl_session);if (status && status != -NFS4ERR_BADSESSION && status != -NFS4ERR_DEADSESSION ) {
include/linux/nfs_fs_sb.h +1 -0
@@ -209,6 +209,7 @@ struct nfs4_session {unsigned long session_state;u32 hash_alg;u32 ssv_len; +struct completion complete;/* The fore and back channel */ struct nfs4_channel_attrs fc_attrs;
Signature · 1 line
1.6.0.6
Re: [PATCH 2/2] nfs41: fix state manager deadlock in session reset
Quoted 3 lines · On Wed, Nov 11, 2009 at 2:20 PM, <[email protected]> wrote:
From: Andy Adamson <[email protected]>
If the session is reset during state recovery, the state manager thre=
ad can
Quoted 3 lines
sleep on the
slot_tbl_waitqcausing a deadlock.Add a completion framework to the session. =A0Have the state manager =
thread set
Quoted 1 line
a new session state (
NFS4CLNT_SESSION_DRAINING) and wait for the sess=
ion slot
Quoted 10 lines
table to drain.
Signal the state manager thread in
nfs41_sequence_free_slotwhen theNFS4CLNT_SESSION_DRAININGbit is set and the session is drained.Reported-by Trond Myklebust <[email protected]>Signed-off-by Andy Adamson <[email protected]>---
=A0fs/nfs/nfs4_fs.h =A0 =A0 =A0 =A0 =A0| =A0 =A01 + =A0fs/nfs/nfs4proc.c =A0 =A0 =A0 =A0 | =A0 25 ++++++++++++++++-------=
--
fs/nfs/nfs4proc.c void nfs41_sequence_free_slot(const struct nfs_c=
Quoted 21 lines
=A0fs/nfs/nfs4state.c =A0 =A0 =A0 =A0| =A0 15 +++++++++++++++ =A0include/linux/nfs_fs_sb.h | =A0 =A01 +
=A04 files changed, 33 insertions(+), 9 deletions(-)fs/nfs/nfs4_fs.h +1
-0@@ -45,6 +45,7 @@ enum nfs4_client_state { =A0 =A0 =A0 =A0NFS4CLNT_RECLAIM_NOGRACE , =A0 =A0 =A0 =A0NFS4CLNT_DELEGRETURN, =A0 =A0 =A0 =A0NFS4CLNT_SESSION_SETUP, + =A0 =A0 =A0 NFS4CLNT_SESSION_DRAINING, =A0};=A0/*
fs/nfs/nfs4proc.c +0
-0@@ -355,6 +355,16 @@ void nfs41_sequence_free_slot(const struct nfs_c=
lient *clp,
fs/nfs/nfs4proc.c void nfs41_sequence_free_slot(const struct nfs_c= · line 360
Quoted 6 lines
fs/nfs/nfs4proc.c +3
-0=A0 =A0 =A0 =A0} =A0 =A0 =A0 =A0nfs4_free_slot (tbl, res->sr_slotid); =A0 =A0 =A0 =A0res->sr_slotid =3D NFS4_MAX_SLOT_TABLE; + + =A0 =A0 =A0 /* Signal state manager thread if session is drained */ + =A0 =A0 =A0 if (test_bit(NFS4CLNT_SESSION_DRAINING, &clp->cl_state)=
) {
fs/nfs/nfs4proc.c void nfs41_sequence_free_slot(const struct nfs_c= · line 362
Quoted 2 lines
fs/nfs/nfs4proc.c +2
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_lock(&tbl->slot_tbl_lock); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (tbl->highest_used_slotid =3D=3D -1)=
{
fs/nfs/nfs4proc.c void nfs41_sequence_free_slot(const struct nfs_c= · line 363
Quoted 1 line
fs/nfs/nfs4proc.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s COMPLETE: S=
ession Drained\n", __func__);
fs/nfs/nfs4proc.c void nfs41_sequence_free_slot(const struct nfs_c= · line 364
Quoted 1 line
fs/nfs/nfs4proc.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 complete(&clp->cl_sessi=
on->complete);
fs/nfs/nfs4proc.c static int nfs41_setup_sequence(struct nfs4_ses=
Quoted 7 lines
fs/nfs/nfs4proc.c +3
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 } + =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&tbl->slot_tbl_lock); + =A0 =A0 =A0 } =A0} =A0static void nfs41_sequence_done(struct nfs_client *clp,@@ -448,15 +458,11 @@ static int nfs41_setup_sequence(struct nfs4_ses=
sion *session,
Quoted 3 lines
fs/nfs/nfs4proc.c +0
-0=A0 =A0 =A0 =A0spin_lock(&tbl->slot_tbl_lock);=A0 =A0 =A0 =A0if (test_bit(NFS4CLNT_SESSION_SETUP, &session->clp->cl=
_state)) {
Quoted 2 lines
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (
tbl->highest_used_slotid!=3D -1) {
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 rpc_sleep_on(&tbl->slot=
_tbl_waitq, task, NULL);
Quoted 1 line
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&tbl->slot_=
tbl_lock);
Quoted 1 line
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 dprintk("<-- %s: Sessio=
n reset: draining\n", __func__);
Quoted 4 lines
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 return
-EAGAIN;
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 }
-
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 /* The slot table is empty; start the r=
eset thread */
Quoted 1 line
- =A0 =A0 =A0 =A0 =A0 =A0 =A0
dprintk("%s Session Reset\n", __func__)=
;
Quoted 2 lines
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 /*
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0* The state manager will wait until =
the slot table is empty.
Quoted 3 lines
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0* Schedule the reset thread
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0*/
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s Schedule Session Reset\n", =
__func__);
Quoted 1 line
=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0rpc_sleep_on(&tbl->slot_tbl_waitq, tas=
k, NULL);
Quoted 1 line
=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0nfs4_schedule_state_manager(session->c=
lp);
fs/nfs/nfs4proc.c struct nfs4_session *nfs4_alloc_session(struct =
Quoted 2 lines
=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0spin_unlock(&tbl->slot_tbl_lock);
fs/nfs/nfs4proc.c +0
-0@@ -4583,6 +4589,7 @@ struct nfs4_session *nfs4_alloc_session(struct =
nfs_client *clp)
fs/nfs/nfs4state.c static void nfs4_session_recovery_handle_error=
Quoted 12 lines
=A0 =A0 =A0 =A0 *
nfs_clientstruct
=A0 =A0 =A0 =A0 */
=A0 =A0 =A0 =A0clp->cl_cons_state =3DNFS_CS_SESSION_INITING;
+ =A0 =A0 =A0init_completion(&session->complete);=A0 =A0 =A0 =A0tbl =3D
&session->fc_slot_table; =A0 =A0 =A0 =A0spin_lock_init(&tbl->slot_tbl_lock);fs/nfs/nfs4state.c +0
-0@@ -1162,8 +1162,23 @@ static void nfs4_session_recovery_handle_error=
(struct nfs_client *clp, int err)
fs/nfs/nfs4state.c static void nfs4_session_recovery_handle_error= · line 1170
Quoted 10 lines
fs/nfs/nfs4state.c +5
-0=A0static int nfs4_reset_session(struct nfs_client *clp) =A0{ + =A0 =A0 =A0 struct nfs4_session *ses =3D clp->cl_session; + =A0 =A0 =A0 struct nfs4_slot_table *tbl =3D &ses->fc_slot_table; =A0 =A0 =A0 =A0int status; + =A0 =A0 =A0 spin_lock(&tbl->slot_tbl_lock); + =A0 =A0 =A0 if (tbl->highest_used_slotid !=3D -1) { + =A0 =A0 =A0 =A0 =A0 =A0 =A0 set_bit(NFS4CLNT_SESSION_DRAINING, &clp=
->cl_state);
fs/nfs/nfs4state.c static void nfs4_session_recovery_handle_error= · line 1172
Quoted 2 lines
fs/nfs/nfs4state.c +2
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&tbl->slot_tbl_lock); + =A0 =A0 =A0 =A0 =A0 =A0 =A0 status =3D wait_for_completion_interrup=
tible(&ses->complete);
fs/nfs/nfs4state.c static void nfs4_session_recovery_handle_error= · line 1173
Quoted 1 line
fs/nfs/nfs4state.c +1
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 clear_bit(NFS4CLNT_SESSION_DRAINING, &c=
lp->cl_state);
fs/nfs/nfs4state.c static void nfs4_session_recovery_handle_error= · line 1176
Quoted 3 lines
fs/nfs/nfs4state.c +3
-0+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (status) /* -ERESTARTSYS */ + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 goto out; + =A0 =A0 =A0 =A0 =A0 =A0 =A0 INIT_COMPLETION(ses->complete);
Looking at my own code - I need to move the INIT_COMPLETION after the wait_for_completion_interruptible and before the status check so that it gets called even on -ERESTARTSYS.
I'll wait for other comments before sending a fix.
-->Andy
Quoted 12 lines
fs/nfs/nfs4state.c +4
-0+ =A0 =A0 =A0 } else { + =A0 =A0 =A0 =A0 =A0 =A0 =A0 spin_unlock(&tbl->slot_tbl_lock); + =A0 =A0 =A0 } + =A0 =A0 =A0 =A0status =3D nfs4_proc_destroy_session(clp->cl_session);=A0 =A0 =A0 =A0if (status && status !=3D -NFS4ERR_BADSESSION && =A0 =A0 =A0 =A0 =A0 =A0status !=3D -NFS4ERR_DEADSESSION) {
include/linux/nfs_fs_sb.h +0
-0@@ -209,6 +209,7 @@ struct nfs4_session {
Quoted 1 line
=A0 =A0 =A0 =A0unsigned long =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 sess=
ion_state;
Quoted 1 line
=A0 =A0 =A0 =A0u32 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0=
=A0 hash_alg;
Quoted 1 line
=A0 =A0 =A0 =A0u32 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0=
=A0 ssv_len;
Quoted 9 lines
+ =A0 =A0 =A0 struct completion =A0 =A0 =A0 =A0 =A0 =A0 =A0 complete;
=A0 =A0 =A0 =A0/* The fore and back channel */
=A0 =A0 =A0 =A0structnfs4_channel_attrs=A0 =A0 =A0fc_attrs; --
1.6.0.6--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" =
in
Quoted 3 lines
the body of a message to [email protected]
More majordomo info at =A0http://vger.kernel.org/majordomo-info.html
A message that is not in the archive (sent off-list or not yet received): 89c397150911111151q7458d1e3m84bf0c39710a8537-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org
Re: [pnfs] [PATCH 2/2] nfs41: fix state manager deadlock in session reset
Quoted 96 lines · William A. (Andy) Adamson
Quoted 95 lines · On Wed, Nov 11, 2009 at 2:20 PM, <[email protected]> wrote:
From: Andy Adamson <[email protected]>
If the session is reset during state recovery, the state manager thread can sleep on the
slot_tbl_waitqcausing a deadlock.Add a completion framework to the session. Have the state manager thread set a new session state (
NFS4CLNT_SESSION_DRAINING) and wait for the session slot table to drain.Signal the state manager thread in
nfs41_sequence_free_slotwhen theNFS4CLNT_SESSION_DRAININGbit is set and the session is drained.Reported-by Trond Myklebust <[email protected]>Signed-off-by Andy Adamson <[email protected]>--- fs/nfs/nfs4_fs.h | 1 + fs/nfs/nfs4proc.c | 25 ++++++++++++++++--------- fs/nfs/nfs4state.c | 15 +++++++++++++++ include/linux/nfs_fs_sb.h | 1 + 4 files changed, 33 insertions(+), 9 deletions(-)fs/nfs/nfs4_fs.h +1
-0@@ -45,6 +45,7 @@ enum nfs4_client_state {NFS4CLNT_RECLAIM_NOGRACE ,NFS4CLNT_DELEGRETURN ,NFS4CLNT_SESSION_SETUP , + NFS4CLNT_SESSION_DRAINING, };/* fs/nfs/nfs4proc.c +16
-9@@ -355,6 +355,16 @@ void nfs41_sequence_free_slot(const struct nfs_client *clp, }nfs4_free_slot (tbl, res->sr_slotid); res->sr_slotid =NFS4_MAX_SLOT_TABLE ; + +/* Signal state manager thread if session is drained */ +if (test_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state)) { +spin_lock (&tbl->slot_tbl_lock); +if (tbl->highest_used_slotid ==-1 ) { +dprintk ("%s COMPLETE: Session Drained\n" , __func__); +complete (&clp->cl_session->complete); + } +spin_unlock (&tbl->slot_tbl_lock); + } } static void nfs41_sequence_done(struct nfs_client *clp,@@ -448,15 +458,11 @@ static int nfs41_setup_sequence(struct nfs4_session *session,spin_lock (&tbl->slot_tbl_lock);if (test_bit (NFS4CLNT_SESSION_SETUP , &session->clp->cl_state)) { -if (tbl->highest_used_slotid !=-1 ) { -rpc_sleep_on (&tbl->slot_tbl_waitq, task,NULL ); -spin_unlock (&tbl->slot_tbl_lock); -dprintk ("<-- %s: Session reset: draining\n" , __func__); -return -EAGAIN ; - } - -/* The slot table is empty; start the reset thread */ -dprintk ("%s Session Reset\n" , __func__); +/* +* The state manager will wait until the slot table is empty. +* Schedule the reset thread +*/ +dprintk ("%s Schedule Session Reset\n" , __func__);rpc_sleep_on (&tbl->slot_tbl_waitq, task,NULL );nfs4_schedule_state_manager (session->clp);spin_unlock (&tbl->slot_tbl_lock);@@ -4583,6 +4589,7 @@ struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp)* nfs_client struct */ clp->cl_cons_state =NFS_CS_SESSION_INITING ; +init_completion (&session->complete); tbl = &session->fc_slot_table;spin_lock_init (&tbl->slot_tbl_lock);fs/nfs/nfs4state.c +2
-0@@ -1162,8 +1162,23 @@ static void nfs4_session_recovery_handle_error(struct nfs_client *clp, int err)static int nfs4_reset_session (struct nfs_client *clp) { +struct nfs4_session *ses = clp->cl_session; +struct nfs4_slot_table *tbl = &ses->fc_slot_table;int status;
Quoted 14 lines
fs/nfs/nfs4state.c static void nfs4_session_recovery_handle_error(struct nfs_client *clp, int err) · line 1177
Quoted 9 lines
fs/nfs/nfs4state.c +9
-0+spin_lock (&tbl->slot_tbl_lock); +if (tbl->highest_used_slotid !=-1 ) { +set_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state); +spin_unlock (&tbl->slot_tbl_lock); + status =wait_for_completion_interruptible (&ses->complete); +clear_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state); +if (status)/* -ERESTARTSYS */ + goto out; +INIT_COMPLETION (ses->complete);Looking at my own code - I need to move the
INIT_COMPLETIONafter thewait_for_completion_interruptibleand before the status check so that it gets called even on-ERESTARTSYS.
Might there be a race with nfs41_sequence_free_slot? Can't you just call INIT_COMPLETION under the slot_tbl_lock before calling wait_for_completion_interruptible?
I.e, in nfs41_sequence_free_slot:
}
nfs4_free_slot(tbl, res->sr_slotid);
res->sr_slotid = NFS4_MAX_SLOT_TABLE;
+
+ /* Signal state manager thread if session is drained */
+ if (test_bit(NFS4CLNT_SESSION_DRAINING, &clp->cl_state)) {
+ spin_lock(&tbl->slot_tbl_lock);
+ if (tbl->highest_used_slotid == -1 &&
+ test_bit(NFS4CLNT_SESSION_DRAINING, &clp->cl_state)) {
+ dprintk("%s COMPLETE: Session Drained\n", __func__);
+ complete(&clp->cl_session->complete);
+ }
+ spin_unlock(&tbl->slot_tbl_lock);
+ }
}
and here:
static int nfs4_reset_session(struct nfs_client *clp) {
+ struct nfs4_session *ses = clp->cl_session;
+ struct nfs4_slot_table *tbl = &ses->fc_slot_table;
int status;
+ spin_lock(&tbl->slot_tbl_lock);
+ if (tbl->highest_used_slotid != -1) {
+ set_bit(NFS4CLNT_SESSION_DRAINING, &clp->cl_state);
+ INIT_COMPLETION(ses->complete);
+ spin_unlock(&tbl->slot_tbl_lock);
+ status = wait_for_completion_interruptible(&ses->complete);
+ clear_bit(NFS4CLNT_SESSION_DRAINING, &clp->cl_state);
+ if (status) /* -ERESTARTSYS */
+ goto out;
Benny
Quoted 37 lines
I'll wait for other comments before sending a fix.
-->Andy
Quoted 22 lines
fs/nfs/nfs4state.c +4
-0+ }else { +spin_unlock (&tbl->slot_tbl_lock); + } + status =nfs4_proc_destroy_session (clp->cl_session);if (status && status != -NFS4ERR_BADSESSION && status != -NFS4ERR_DEADSESSION ) {include/linux/nfs_fs_sb.h +1
-1@@ -209,6 +209,7 @@ struct nfs4_session {unsigned long session_state;u32 hash_alg;u32 ssv_len; +struct completion complete;/* The fore and back channel */ struct nfs4_channel_attrs fc_attrs; --1.6.0.6
Quoted 5 lines
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html_______________________________________________
pNFS mailing list
[email protected]
http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs
Re: [pnfs] [PATCH 2/2] nfs41: fix state manager deadlock in session reset
Quoted 151 lines · Benny Halevy
Quoted 96 lines · William A. (Andy) Adamson
Quoted 95 lines · On Wed, Nov 11, 2009 at 2:20 PM, <[email protected]> wrote:
From: Andy Adamson <[email protected]>
If the session is reset during state recovery, the state manager thread can sleep on the
slot_tbl_waitqcausing a deadlock.Add a completion framework to the session. Have the state manager thread set a new session state (
NFS4CLNT_SESSION_DRAINING) and wait for the session slot table to drain.Signal the state manager thread in
nfs41_sequence_free_slotwhen theNFS4CLNT_SESSION_DRAININGbit is set and the session is drained.Reported-by Trond Myklebust <[email protected]>Signed-off-by Andy Adamson <[email protected]>--- fs/nfs/nfs4_fs.h | 1 + fs/nfs/nfs4proc.c | 25 ++++++++++++++++--------- fs/nfs/nfs4state.c | 15 +++++++++++++++ include/linux/nfs_fs_sb.h | 1 + 4 files changed, 33 insertions(+), 9 deletions(-)fs/nfs/nfs4_fs.h +1
-0@@ -45,6 +45,7 @@ enum nfs4_client_state {NFS4CLNT_RECLAIM_NOGRACE ,NFS4CLNT_DELEGRETURN ,NFS4CLNT_SESSION_SETUP , + NFS4CLNT_SESSION_DRAINING, };/* fs/nfs/nfs4proc.c +16
-9@@ -355,6 +355,16 @@ void nfs41_sequence_free_slot(const struct nfs_client *clp, }nfs4_free_slot (tbl, res->sr_slotid); res->sr_slotid =NFS4_MAX_SLOT_TABLE ; + +/* Signal state manager thread if session is drained */ +if (test_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state)) { +spin_lock (&tbl->slot_tbl_lock); +if (tbl->highest_used_slotid ==-1 ) { +dprintk ("%s COMPLETE: Session Drained\n" , __func__); +complete (&clp->cl_session->complete); + } +spin_unlock (&tbl->slot_tbl_lock); + } } static void nfs41_sequence_done(struct nfs_client *clp,@@ -448,15 +458,11 @@ static int nfs41_setup_sequence(struct nfs4_session *session,spin_lock (&tbl->slot_tbl_lock);if (test_bit (NFS4CLNT_SESSION_SETUP , &session->clp->cl_state)) { -if (tbl->highest_used_slotid !=-1 ) { -rpc_sleep_on (&tbl->slot_tbl_waitq, task,NULL ); -spin_unlock (&tbl->slot_tbl_lock); -dprintk ("<-- %s: Session reset: draining\n" , __func__); -return -EAGAIN ; - } - -/* The slot table is empty; start the reset thread */ -dprintk ("%s Session Reset\n" , __func__); +/* +* The state manager will wait until the slot table is empty. +* Schedule the reset thread +*/ +dprintk ("%s Schedule Session Reset\n" , __func__);rpc_sleep_on (&tbl->slot_tbl_waitq, task,NULL );nfs4_schedule_state_manager (session->clp);spin_unlock (&tbl->slot_tbl_lock);@@ -4583,6 +4589,7 @@ struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp)* nfs_client struct */ clp->cl_cons_state =NFS_CS_SESSION_INITING ; +init_completion (&session->complete); tbl = &session->fc_slot_table;spin_lock_init (&tbl->slot_tbl_lock);fs/nfs/nfs4state.c +2
-0@@ -1162,8 +1162,23 @@ static void nfs4_session_recovery_handle_error(struct nfs_client *clp, int err)static int nfs4_reset_session (struct nfs_client *clp) { +struct nfs4_session *ses = clp->cl_session; +struct nfs4_slot_table *tbl = &ses->fc_slot_table;int status;Quoted 14 lines
fs/nfs/nfs4state.c static void nfs4_session_recovery_handle_error(struct nfs_client *clp, int err) · line 1177
Quoted 9 lines
fs/nfs/nfs4state.c +9
-0+spin_lock (&tbl->slot_tbl_lock); +if (tbl->highest_used_slotid !=-1 ) { +set_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state); +spin_unlock (&tbl->slot_tbl_lock); + status =wait_for_completion_interruptible (&ses->complete); +clear_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state); +if (status)/* -ERESTARTSYS */ + goto out; +INIT_COMPLETION (ses->complete);Looking at my own code - I need to move the
INIT_COMPLETIONafter thewait_for_completion_interruptibleand before the status check so that it gets called even on-ERESTARTSYS.Might there be a race with
nfs41_sequence_free_slot? Can't you just callINIT_COMPLETIONunder theslot_tbl_lockbefore callingwait_for_completion_interruptible?I.e, in
nfs41_sequence_free_slot:fs/nfs/nfs4proc.c +11
-0}nfs4_free_slot (tbl, res->sr_slotid); res->sr_slotid =NFS4_MAX_SLOT_TABLE ; + +/* Signal state manager thread if session is drained */ +if (test_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state)) { +spin_lock (&tbl->slot_tbl_lock); +if (tbl->highest_used_slotid ==-1 && +test_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state)) { +dprintk ("%s COMPLETE: Session Drained\n" , __func__); +complete (&clp->cl_session->complete); + } +spin_unlock (&tbl->slot_tbl_lock); + } }and here:
static int
nfs4_reset_session(struct nfs_client *clp){fs/nfs/nfs4state.c +11
-0+struct nfs4_session *ses = clp->cl_session; +struct nfs4_slot_table *tbl = &ses->fc_slot_table;int status; +spin_lock (&tbl->slot_tbl_lock); +if (tbl->highest_used_slotid !=-1 ) { +set_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state); +INIT_COMPLETION (ses->complete); +spin_unlock (&tbl->slot_tbl_lock); + status =wait_for_completion_interruptible (&ses->complete); +clear_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state); +if (status)/* -ERESTARTSYS */ + goto out;
Quoted 1 line
Benny
You shouldn't need to call it under the spinlock. Just remember to call INIT_COMPLETION before you set NFS4CLNT_SESSION_DRAINING.
IOW:
INIT_COMPLETION(ses->complete);
spin_lock(&tbl->slot_tbl_lock);
if (tbl->highest_used_slotid != -1) {
set_bit(NFS4CLNT_SESSION_DRAINING, &clp->cl_state);
....Trond
Signature · 6 lines
Trond Myklebust Linux NFS client maintainer NetApp [email protected] www.netapp.com
A message that is not in the archive (sent off-list or not yet received): 1258033600.2968.16.camel-rJ7iovZKK19ZJLDQqaL3InhyD016LWXt@public.gmane.org
Re: [pnfs] [PATCH 2/2] nfs41: fix state manager deadlock in session reset
Quoted 157 lines · Trond Myklebust
Quoted 151 lines · Benny Halevy
Quoted 96 lines · William A. (Andy) Adamson
Quoted 95 lines · On Wed, Nov 11, 2009 at 2:20 PM, <[email protected]> wrote:
From: Andy Adamson <[email protected]>
If the session is reset during state recovery, the state manager thread can sleep on the
slot_tbl_waitqcausing a deadlock.Add a completion framework to the session. Have the state manager thread set a new session state (
NFS4CLNT_SESSION_DRAINING) and wait for the session slot table to drain.Signal the state manager thread in
nfs41_sequence_free_slotwhen theNFS4CLNT_SESSION_DRAININGbit is set and the session is drained.Reported-by Trond Myklebust <[email protected]>Signed-off-by Andy Adamson <[email protected]>--- fs/nfs/nfs4_fs.h | 1 + fs/nfs/nfs4proc.c | 25 ++++++++++++++++--------- fs/nfs/nfs4state.c | 15 +++++++++++++++ include/linux/nfs_fs_sb.h | 1 + 4 files changed, 33 insertions(+), 9 deletions(-)fs/nfs/nfs4_fs.h +1
-0@@ -45,6 +45,7 @@ enum nfs4_client_state {NFS4CLNT_RECLAIM_NOGRACE ,NFS4CLNT_DELEGRETURN ,NFS4CLNT_SESSION_SETUP , + NFS4CLNT_SESSION_DRAINING, };/* fs/nfs/nfs4proc.c +16
-9@@ -355,6 +355,16 @@ void nfs41_sequence_free_slot(const struct nfs_client *clp, }nfs4_free_slot (tbl, res->sr_slotid); res->sr_slotid =NFS4_MAX_SLOT_TABLE ; + +/* Signal state manager thread if session is drained */ +if (test_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state)) { +spin_lock (&tbl->slot_tbl_lock); +if (tbl->highest_used_slotid ==-1 ) { +dprintk ("%s COMPLETE: Session Drained\n" , __func__); +complete (&clp->cl_session->complete); + } +spin_unlock (&tbl->slot_tbl_lock); + } } static void nfs41_sequence_done(struct nfs_client *clp,@@ -448,15 +458,11 @@ static int nfs41_setup_sequence(struct nfs4_session *session,spin_lock (&tbl->slot_tbl_lock);if (test_bit (NFS4CLNT_SESSION_SETUP , &session->clp->cl_state)) { -if (tbl->highest_used_slotid !=-1 ) { -rpc_sleep_on (&tbl->slot_tbl_waitq, task,NULL ); -spin_unlock (&tbl->slot_tbl_lock); -dprintk ("<-- %s: Session reset: draining\n" , __func__); -return -EAGAIN ; - } - -/* The slot table is empty; start the reset thread */ -dprintk ("%s Session Reset\n" , __func__); +/* +* The state manager will wait until the slot table is empty. +* Schedule the reset thread +*/ +dprintk ("%s Schedule Session Reset\n" , __func__);rpc_sleep_on (&tbl->slot_tbl_waitq, task,NULL );nfs4_schedule_state_manager (session->clp);spin_unlock (&tbl->slot_tbl_lock);@@ -4583,6 +4589,7 @@ struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp)* nfs_client struct */ clp->cl_cons_state =NFS_CS_SESSION_INITING ; +init_completion (&session->complete); tbl = &session->fc_slot_table;spin_lock_init (&tbl->slot_tbl_lock);fs/nfs/nfs4state.c +2
-0@@ -1162,8 +1162,23 @@ static void nfs4_session_recovery_handle_error(struct nfs_client *clp, int err)static int nfs4_reset_session (struct nfs_client *clp) { +struct nfs4_session *ses = clp->cl_session; +struct nfs4_slot_table *tbl = &ses->fc_slot_table;int status;Quoted 14 lines
fs/nfs/nfs4state.c static void nfs4_session_recovery_handle_error(struct nfs_client *clp, int err) · line 1177
Quoted 9 lines
fs/nfs/nfs4state.c +9
-0+spin_lock (&tbl->slot_tbl_lock); +if (tbl->highest_used_slotid !=-1 ) { +set_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state); +spin_unlock (&tbl->slot_tbl_lock); + status =wait_for_completion_interruptible (&ses->complete); +clear_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state); +if (status)/* -ERESTARTSYS */ + goto out; +INIT_COMPLETION (ses->complete);Looking at my own code - I need to move the
INIT_COMPLETIONafter thewait_for_completion_interruptibleand before the status check so that it gets called even on-ERESTARTSYS.Might there be a race with
nfs41_sequence_free_slot? Can't you just callINIT_COMPLETIONunder theslot_tbl_lockbefore callingwait_for_completion_interruptible?I.e, in
nfs41_sequence_free_slot:} nfs4_free_slot(tbl, res->sr_slotid); res->sr_slotid = NFS4_MAX_SLOT_TABLE;fs/nfs/nfs4proc.c +11
-0+ +/* Signal state manager thread if session is drained */ +if (test_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state)) { +spin_lock (&tbl->slot_tbl_lock); +if (tbl->highest_used_slotid ==-1 && +test_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state)) { +dprintk ("%s COMPLETE: Session Drained\n" , __func__); +complete (&clp->cl_session->complete); + } +spin_unlock (&tbl->slot_tbl_lock); + } }and here:
static int
nfs4_reset_session(struct nfs_client *clp){fs/nfs/nfs4state.c +11
-0+struct nfs4_session *ses = clp->cl_session; +struct nfs4_slot_table *tbl = &ses->fc_slot_table;int status; +spin_lock (&tbl->slot_tbl_lock); +if (tbl->highest_used_slotid !=-1 ) { +set_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state); +INIT_COMPLETION (ses->complete); +spin_unlock (&tbl->slot_tbl_lock); + status =wait_for_completion_interruptible (&ses->complete); +clear_bit (NFS4CLNT_SESSION_DRAINING , &clp->cl_state); +if (status)/* -ERESTARTSYS */ + goto out;Quoted 1 line
Benny
You shouldn't need to call it under the spinlock. Just remember to call
INIT_COMPLETIONbefore you setNFS4CLNT_SESSION_DRAINING.
Quoted 8 lines
IOW:
INIT_COMPLETION(ses->complete); spin_lock(&tbl->slot_tbl_lock); if (tbl->highest_used_slotid != -1) { set_bit(NFS4CLNT_SESSION_DRAINING, &clp->cl_state); ....
Yes - I see now. INIT_COMPLETION shoud be called before wait_for_completion_XXX, not after.
-->Andy
Quoted 10 lines
Trond
--
Trond Myklebust
Linux NFS client maintainerNetApp
[email protected]
www.netapp.com