nfs41: resolve some race conditions with queued SEQUENCE operations when unmounting
Subsystems (2)
PATCH 1/1 nfs41: resolve some race conditions with queued SEQUENCE operations when unmounting
--- fs/nfs/nfs4_fs.h | 1 + fs/nfs/nfs4proc.c | 3 +++ fs/nfs/nfs4state.c | 11 ++++++++--- 3 files changed, 12 insertions(+), 3 deletions(-)
fs/nfs/nfs4_fs.h +1 -0
@@ -45,6 +45,7 @@ enum nfs4_client_state {NFS4CLNT_RECLAIM_NOGRACE ,NFS4CLNT_DELEGRETURN ,NFS4CLNT_SESSION_SETUP , + NFS4CLNT_SESSION_DESTROY, };/*
fs/nfs/nfs4proc.c +3 -0
@@ -359,6 +359,8 @@ static void nfs41_sequence_done(struct nfs_client *clp,struct nfs4_slot_table *tbl;struct nfs4_slot *slot; +if (!nfs4_has_session (clp)) + goto out;/* * sr_status remains 1 if an RPC level error occurred. The server * may or may not have processed the sequence operation..
@@ -4616,6 +4618,7 @@ struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp)void nfs4_destroy_session (struct nfs4_session *session) { +set_bit (NFS4CLNT_SESSION_DESTROY , &session->clp->cl_state);nfs4_proc_destroy_session (session);dprintk ("%s Destroy backchannel for xprt %p\n" , __func__, session->clp->cl_rpcclient->cl_xprt);
fs/nfs/nfs4state.c +8 -3
@@ -1253,11 +1253,16 @@ static void nfs4_state_manager(struct nfs_client *clp) }/* Initialize or reset the session */ if (test_and_clear_bit (NFS4CLNT_SESSION_SETUP , &clp->cl_state) - &&nfs4_has_session (clp)) { + &&nfs4_has_session (clp)) {if (clp->cl_cons_state ==NFS_CS_SESSION_INITING ) status =nfs4_initialize_session (clp); -else - status =nfs4_reset_session (clp); +else { +if (test_bit (NFS4CLNT_SESSION_DESTROY , + &clp->cl_state)) + status = -EIO ; +else + status =nfs4_reset_session (clp); + }if (status) {if (status == -NFS4ERR_STALE_CLIENTID )continue ;
Signature · 1 line
1.6.2.5
Quoted 54 lines · Alexandros Batsakis
Signed-off-by Alexandros Batsakis <[email protected]>--- fs/nfs/nfs4_fs.h | 1 + fs/nfs/nfs4proc.c | 3 +++ fs/nfs/nfs4state.c | 11 ++++++++--- 3 files changed, 12 insertions(+), 3 deletions(-)fs/nfs/nfs4_fs.h +1
-0@@ -45,6 +45,7 @@ enum nfs4_client_state {NFS4CLNT_RECLAIM_NOGRACE ,NFS4CLNT_DELEGRETURN ,NFS4CLNT_SESSION_SETUP , + NFS4CLNT_SESSION_DESTROY, };/* fs/nfs/nfs4proc.c +3
-0@@ -359,6 +359,8 @@ static void nfs41_sequence_done(struct nfs_client *clp,struct nfs4_slot_table *tbl;struct nfs4_slot *slot; +if (!nfs4_has_session (clp)) + goto out;/* * sr_status remains 1 if an RPC level error occurred. The server * may or may not have processed the sequence operation.. @@ -4616,6 +4618,7 @@ struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp)void nfs4_destroy_session (struct nfs4_session *session) { +set_bit (NFS4CLNT_SESSION_DESTROY , &session->clp->cl_state);nfs4_proc_destroy_session (session);dprintk ("%s Destroy backchannel for xprt %p\n" , __func__, session->clp->cl_rpcclient->cl_xprt);fs/nfs/nfs4state.c +1
-3@@ -1253,11 +1253,16 @@ static void nfs4_state_manager(struct nfs_client *clp) }/* Initialize or reset the session */ if (test_and_clear_bit (NFS4CLNT_SESSION_SETUP , &clp->cl_state) - &&nfs4_has_session (clp)) { + &&nfs4_has_session (clp)) {if (clp->cl_cons_state ==NFS_CS_SESSION_INITING ) status =nfs4_initialize_session (clp); -else - status =nfs4_reset_session (clp);
fs/nfs/nfs4state.c static void nfs4_state_manager(struct nfs_client *clp) · line 1268
Quoted 10 lines
fs/nfs/nfs4state.c +7
-0+else { +if (test_bit (NFS4CLNT_SESSION_DESTROY , + &clp->cl_state)) + status = -EIO ; +else + status =nfs4_reset_session (clp); + }if (status) {if (status == -NFS4ERR_STALE_CLIENTID )continue ;
Wait... Why is all this needed?
AFAICS, nfs_destroy_session() is only called by nfs_free_client() once the very last reference to the nfs_client has been released. Now since the state manager keeps a reference to the nfs_client all the time while it is running (see nfs4_run_state_manager()), we _can't_ have a race between it and nfs4_destroy_session().
A message that is not in the archive (sent off-list or not yet received): 1255555809.6308.34.camel-rJ7iovZKK19ZJLDQqaL3InhyD016LWXt@public.gmane.org
Quoted 19 lines · Trond Myklebust
Quoted 18 lines · Alexandros Batsakis
Signed-off-by Alexandros Batsakis <[email protected]>---
=A0fs/nfs/nfs4_fs.h =A0 | =A0 =A01 + =A0fs/nfs/nfs4proc.c =A0| =A0 =A03 +++ =A0fs/nfs/nfs4state.c | =A0 11 ++++++++--- =A03 files changed, 12 insertions(+), 3 deletions(-)fs/nfs/nfs4_fs.h +1
-0@@ -45,6 +45,7 @@ enum nfs4_client_state { =A0 =A0 =A0 NFS4CLNT_RECLAIM_NOGRACE, =A0 =A0 =A0 NFS4CLNT_DELEGRETURN, =A0 =A0 =A0 NFS4CLNT_SESSION_SETUP, + =A0 =A0 NFS4CLNT_SESSION_DESTROY, =A0};
Quoted 6 lines
fs/nfs/nfs4proc.c static void nfs41_sequence_done(struct nfs_clien=
Quoted 6 lines
=A0/*
fs/nfs/nfs4proc.c +0
-0@@ -359,6 +359,8 @@ static void nfs41_sequence_done(struct nfs_clien=
t *clp,
Quoted 7 lines
Quoted 7 lines
fs/nfs/nfs4proc.c +2
-0=A0 =A0 =A0 struct nfs4_slot_table *tbl; =A0 =A0 =A0 struct nfs4_slot *slot; + =A0 =A0 if (!nfs4_has_session(clp)) + =A0 =A0 =A0 =A0 =A0 =A0 goto out;=A0 =A0 =A0 /*
=A0 =A0 =A0 =A0*sr_statusremains 1 if an RPC level error occurred.=
The server
Quoted 1 line
Quoted 1 line
=A0 =A0 =A0 =A0* may or may not have processed the sequence operatio=
n..
Quoted 1 line
fs/nfs/nfs4proc.c struct nfs4_session *nfs4_alloc_session(struct=
Quoted 1 line
fs/nfs/nfs4proc.c +0
-0@@ -4616,6 +4618,7 @@ struct nfs4_session *nfs4_alloc_session(struct=
nfs_client *clp)
Quoted 4 lines
fs/nfs/nfs4proc.c struct nfs4_session *nfs4_alloc_session(struct= · line 4620
Quoted 4 lines
fs/nfs/nfs4proc.c +1
-0=A0void nfs4_destroy_session(struct nfs4_session *session) =A0{ + =A0 =A0 set_bit(NFS4CLNT_SESSION_DESTROY, &session->clp->cl_state)=
;
Quoted 3 lines
Quoted 3 lines
fs/nfs/nfs4proc.c +0
-0=A0 =A0 =A0 nfs4_proc_destroy_session(session); =A0 =A0 =A0 dprintk("%s Destroy backchannel for xprt %p\n",=A0 =A0 =A0 =A0 =A0 =A0 =A0
__func__, session->clp->cl_rpcclient->cl=
_xprt);
Quoted 5 lines
fs/nfs/nfs4state.c static void nfs4_state_manager(struct nfs_cl=
Quoted 5 lines
fs/nfs/nfs4state.c +0
-0@@ -1253,11 +1253,16 @@ static void nfs4_state_manager(struct nfs_cl=
ient *clp)
Quoted 3 lines
Quoted 3 lines
fs/nfs/nfs4state.c +0
-0=A0 =A0 =A0 =A0 =A0 =A0 =A0 }=A0 =A0 =A0 =A0 =A0 =A0 =A0 /* Initialize or reset the session */ =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (test_and_clear_bit(NFS4CLNT_SESSION_=
SETUP, &clp->cl_state)
Quoted 3 lines
Quoted 3 lines
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0&&
nfs4_has_session(clp)) {
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 &&nfs4_has_session(clp)) { =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (clp->cl_cons_state=3D=
=3D NFS_CS_SESSION_INITING)
Quoted 1 line
Quoted 1 line
=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 status =3D=
nfs4_initialize_session(clp);
Quoted 2 lines
Quoted 2 lines
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 else
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 status =3D=
nfs4_reset_session(clp);
Quoted 2 lines
Quoted 2 lines
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 else {
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (test_b=
it(NFS4CLNT_SESSION_DESTROY,
Quoted 1 line
Quoted 1 line
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0=
=A0 =A0 =A0 =A0&clp->cl_state))
Quoted 1 line
Quoted 1 line
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0=
=A0 status =3D -EIO;
Quoted 2 lines
Quoted 2 lines
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 else
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0=
=A0 status =3D nfs4_reset_session(clp);
Quoted 3 lines
Quoted 3 lines
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 }
=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (status) { =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (stat=
us =3D=3D -NFS4ERR_STALE_CLIENTID)
Quoted 1 line
Quoted 1 line
=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =
=A0 =A0 continue;
Quoted 4 lines
Wait... Why is all this needed?
AFAICS,
nfs_destroy_session()is only called bynfs_free_client()onc=
e
Quoted 1 line
the very last reference to the
nfs_clienthas been released. Now sinc=
e
Quoted 1 line
the state manager keeps a reference to the
nfs_clientall the time wh=
ile
Quoted 3 lines
it is running (see
nfs4_run_state_manager()), we _can't_ have a race between it andnfs4_destroy_session().
This is correct very and in the normal case it is not a problem. However, the problems start when the server is down/rebooted/etc and the client tries to umount.
Then, the (SEQUENCE) requests are _already_ queued and when the server comes up or the request is cancelled (ctl-c), we see these race conditions:
a) nfs41_sequence_done() called after destroy_session() that leads to a NULL pointer dereference
b) a BADSESSION reply to a sequence operation triggers a reset_session() at the same time with destroy_session() (called by umount) that leads to another NULL pointer dereference.
I can send you the call traces if you need more info. I understand that these are pretty rare cases, but they shouldn't lead to a crash.
-alexandros
Quoted 5 lines
_______________________________________________
pNFS mailing list
[email protected]
http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs
A message that is not in the archive (sent off-list or not yet received): 5e24e8930910141450h11e677bbr17ebe3441a8742d8-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org
Quoted 5 lines · Alexandros Batsakis
a)
nfs41_sequence_done()called afterdestroy_session()that leads to a NULL pointer dereference
b) a BADSESSION reply to a sequence operation triggers areset_session()at the same time withdestroy_session()(called by umount) that leads to another NULL pointer dereference.
This would mean that nfs41_sequence_done is being called _after_ the nfs_client (and hence the session) has been destroyed. That sounds like the real bug that needs to be fixed.
Cheers
Trond
A message that is not in the archive (sent off-list or not yet received): 1255557198.6308.36.camel-rJ7iovZKK19ZJLDQqaL3InhyD016LWXt@public.gmane.org
Quoted 10 lines · Trond Myklebust
Quoted 5 lines · Alexandros Batsakis
a)
nfs41_sequence_done()called afterdestroy_session()that leads to a NULL pointer dereference
b) a BADSESSION reply to a sequence operation triggers areset_session()at the same time withdestroy_session()(called by umount) that leads to another NULL pointer dereference.This would mean that
nfs41_sequence_doneis being called _after_ thenfs_client(and hence the session) has been destroyed. That sounds like the real bug that needs to be fixed.
Correction: it means that nfs41_sequence_done is being called after the superblock that "owns" those rpc calls has been destroyed. (Which is a bug... :-))
Cheers
Trond
A message that is not in the archive (sent off-list or not yet received): 1255557476.6308.39.camel-rJ7iovZKK19ZJLDQqaL3InhyD016LWXt@public.gmane.org
Quoted 3 lines · Trond Myklebust
Quoted 2 lines · Trond Myklebust
Quoted 1 line · Alexandros Batsakis
a)
nfs41_sequence_done()called afterdestroy_session()that leads=
to
Quoted 7 lines
Quoted 7 lines
Quoted 4 lines
a NULL pointer dereference
b) a BADSESSION reply to a sequence operation triggers areset_session()at the same time withdestroy_session()(called by umount) that leads to another NULL pointer dereference.This would mean that
nfs41_sequence_doneis being called _after_ thenfs_client(and hence the session) has been destroyed. That sounds l=
ike
Quoted 3 lines
Quoted 1 line
the real bug that needs to be fixed.
Correction: it means that
nfs41_sequence_doneis being called after t=
he
Quoted 1 line
superblock that "owns" those rpc calls has been destroyed. (Which is =
a
Quoted 2 lines
bug... :-))
Agreed. FWIW and from a conceptual point of view, the patch above is a bit orthogonal to that as it deals with the problem within the session scope. It treats the umount just as a session destroyer that happens to always destroy the super-block in the current one-session-per-client implementation. The latter may change, but the patch will remain relevant (obviously with few adjustments). Anyway, as long as we fix the bug I am happy :)
-alexandros
Quoted 8 lines
Cheers
=A0Trond_______________________________________________
pNFS mailing list
[email protected]
http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs
A message that is not in the archive (sent off-list or not yet received): 5e24e8930910141509j3fad98afr66e25c465ced1e42-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org
Quoted 19 lines · Alexandros Batsakis
Quoted 8 lines · Trond Myklebust
Quoted 7 lines · Trond Myklebust
Quoted 5 lines · Alexandros Batsakis
a)
nfs41_sequence_done()called afterdestroy_session()that leads to a NULL pointer dereference
b) a BADSESSION reply to a sequence operation triggers areset_session()at the same time withdestroy_session()(called by umount) that leads to another NULL pointer dereference.Quoted 8 lines
Quoted 3 lines
This would mean that
nfs41_sequence_doneis being called _after_ thenfs_client(and hence the session) has been destroyed. That sounds like the real bug that needs to be fixed.Correction: it means that
nfs41_sequence_doneis being called after the superblock that "owns" those rpc calls has been destroyed. (Which is a bug... :-))
Quoted 6 lines
Agreed. FWIW and from a conceptual point of view, the patch above is a bit orthogonal to that as it deals with the problem within the session scope. It treats the umount just as a session destroyer that happens to always destroy the super-block in the current one-session-per-client implementation. The latter may change, but the patch will remain relevant (obviously with few adjustments).
I completely disagree. If your code may end up starting the state manager after the nfs_client is destroyed, then you _will_ corrupt significant swathes of memory.
Quoted 1 line
Anyway, as long as we fix the bug I am happy :)
I don't think you have...
Trond