Skip to contentSkip to conversation

nfs41: resolve some race conditions with queued SEQUENCE operations when unmounting

PATCH In review Alexandros Batsakis 3 participants 7 messages Oct 14, 2009 5 messages not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 nfs41: resolve some race conditions with queued SEQUENCE operations when unmounting

---
 fs/nfs/nfs4_fs.h   |    1 +
 fs/nfs/nfs4proc.c  |    3 +++
 fs/nfs/nfs4state.c |   11 ++++++++---
 3 files changed, 12 insertions(+), 3 deletions(-)
fs/nfs/nfs4_fs.h +1 -0
index 6ea07a3..554af98 100644
@@ -45,6 +45,7 @@ enum nfs4_client_state {
 	NFS4CLNT_RECLAIM_NOGRACE,
 	NFS4CLNT_DELEGRETURN,
 	NFS4CLNT_SESSION_SETUP,
+	NFS4CLNT_SESSION_DESTROY,
 };
 
 /*
fs/nfs/nfs4proc.c +3 -0
index c321002..9eb21d2 100644
@@ -359,6 +359,8 @@ static void nfs41_sequence_done(struct nfs_client *clp,
 	struct nfs4_slot_table *tbl;
 	struct nfs4_slot *slot;
 
+	if (!nfs4_has_session(clp))
+		goto out;
 	/*
 	 * sr_status remains 1 if an RPC level error occurred. The server
 	 * may or may not have processed the sequence operation..
@@ -4616,6 +4618,7 @@ struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp)
 
 void nfs4_destroy_session(struct nfs4_session *session)
 {
+	set_bit(NFS4CLNT_SESSION_DESTROY, &session->clp->cl_state);
 	nfs4_proc_destroy_session(session);
 	dprintk("%s Destroy backchannel for xprt %p\n",
 		__func__, session->clp->cl_rpcclient->cl_xprt);
fs/nfs/nfs4state.c +8 -3
index 1394dfb..a509b26 100644
@@ -1253,11 +1253,16 @@ static void nfs4_state_manager(struct nfs_client *clp)
 		}
 		/* Initialize or reset the session */
 		if (test_and_clear_bit(NFS4CLNT_SESSION_SETUP, &clp->cl_state)
-		   && nfs4_has_session(clp)) {
+		    && nfs4_has_session(clp)) {
 			if (clp->cl_cons_state == NFS_CS_SESSION_INITING)
 				status = nfs4_initialize_session(clp);
-			else
-				status = nfs4_reset_session(clp);
+			else {
+				if (test_bit(NFS4CLNT_SESSION_DESTROY,
+					     &clp->cl_state))
+					status = -EIO;
+				else
+					status = nfs4_reset_session(clp);
+			}
 			if (status) {
 				if (status == -NFS4ERR_STALE_CLIENTID)
 					continue;
Signature · 1 line
1.6.2.5
#1 lore raw
Quoted 54 lines · Alexandros Batsakis
---
 fs/nfs/nfs4_fs.h   |    1 +
 fs/nfs/nfs4proc.c  |    3 +++
 fs/nfs/nfs4state.c |   11 ++++++++---
 3 files changed, 12 insertions(+), 3 deletions(-)
fs/nfs/nfs4_fs.h +1 -0
index 6ea07a3..554af98 100644
@@ -45,6 +45,7 @@ enum nfs4_client_state {
 	NFS4CLNT_RECLAIM_NOGRACE,
 	NFS4CLNT_DELEGRETURN,
 	NFS4CLNT_SESSION_SETUP,
+	NFS4CLNT_SESSION_DESTROY,
 };
 
 /*
fs/nfs/nfs4proc.c +3 -0
index c321002..9eb21d2 100644
@@ -359,6 +359,8 @@ static void nfs41_sequence_done(struct nfs_client *clp,
 	struct nfs4_slot_table *tbl;
 	struct nfs4_slot *slot;
 
+	if (!nfs4_has_session(clp))
+		goto out;
 	/*
 	 * sr_status remains 1 if an RPC level error occurred. The server
 	 * may or may not have processed the sequence operation..
@@ -4616,6 +4618,7 @@ struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp)
 
 void nfs4_destroy_session(struct nfs4_session *session)
 {
+	set_bit(NFS4CLNT_SESSION_DESTROY, &session->clp->cl_state);
 	nfs4_proc_destroy_session(session);
 	dprintk("%s Destroy backchannel for xprt %p\n",
 		__func__, session->clp->cl_rpcclient->cl_xprt);
fs/nfs/nfs4state.c +1 -3
index 1394dfb..a509b26 100644
@@ -1253,11 +1253,16 @@ static void nfs4_state_manager(struct nfs_client *clp)
 		}
 		/* Initialize or reset the session */
 		if (test_and_clear_bit(NFS4CLNT_SESSION_SETUP, &clp->cl_state)
-		   && nfs4_has_session(clp)) {
+		    && nfs4_has_session(clp)) {
 			if (clp->cl_cons_state == NFS_CS_SESSION_INITING)
 				status = nfs4_initialize_session(clp);
-			else
-				status = nfs4_reset_session(clp);

fs/nfs/nfs4state.c static void nfs4_state_manager(struct nfs_client *clp) · line 1268

Quoted 10 lines
fs/nfs/nfs4state.c +7 -0
+			else {
+				if (test_bit(NFS4CLNT_SESSION_DESTROY,
+					     &clp->cl_state))
+					status = -EIO;
+				else
+					status = nfs4_reset_session(clp);
+			}
 			if (status) {
 				if (status == -NFS4ERR_STALE_CLIENTID)
 					continue;

Wait... Why is all this needed?

AFAICS, nfs_destroy_session() is only called by nfs_free_client() once the very last reference to the nfs_client has been released. Now since the state manager keeps a reference to the nfs_client all the time while it is running (see nfs4_run_state_manager()), we _can't_ have a race between it and nfs4_destroy_session().

A message that is not in the archive (sent off-list or not yet received): 1255555809.6308.34.camel-rJ7iovZKK19ZJLDQqaL3InhyD016LWXt@public.gmane.org

#3 lore raw
Quoted 19 lines · Trond Myklebust
Quoted 18 lines · Alexandros Batsakis

---
=A0fs/nfs/nfs4_fs.h =A0 | =A0 =A01 + =A0fs/nfs/nfs4proc.c =A0| =A0 =A03 +++ =A0fs/nfs/nfs4state.c | =A0 11 ++++++++--- =A03 files changed, 12 insertions(+), 3 deletions(-)

fs/nfs/nfs4_fs.h +1 -0
index 6ea07a3..554af98 100644
@@ -45,6 +45,7 @@ enum nfs4_client_state {
 =A0 =A0 =A0 NFS4CLNT_RECLAIM_NOGRACE,
 =A0 =A0 =A0 NFS4CLNT_DELEGRETURN,
 =A0 =A0 =A0 NFS4CLNT_SESSION_SETUP,
+ =A0 =A0 NFS4CLNT_SESSION_DESTROY,
 =A0};
 
Quoted 6 lines

fs/nfs/nfs4proc.c static void nfs41_sequence_done(struct nfs_clien=

Quoted 6 lines

=A0/*

fs/nfs/nfs4proc.c +0 -0
index c321002..9eb21d2 100644
@@ -359,6 +359,8 @@ static void nfs41_sequence_done(struct nfs_clien=

t *clp,

Quoted 7 lines
Quoted 7 lines
fs/nfs/nfs4proc.c +2 -0
 =A0 =A0 =A0 struct nfs4_slot_table *tbl;
 =A0 =A0 =A0 struct nfs4_slot *slot;
 
+ =A0 =A0 if (!nfs4_has_session(clp))
+ =A0 =A0 =A0 =A0 =A0 =A0 goto out;

=A0 =A0 =A0 /*
=A0 =A0 =A0 =A0* sr_status remains 1 if an RPC level error occurred.=

The server

Quoted 1 line
Quoted 1 line

=A0 =A0 =A0 =A0* may or may not have processed the sequence operatio=

n..

Quoted 1 line

fs/nfs/nfs4proc.c struct nfs4_session *nfs4_alloc_session(struct=

Quoted 1 line
fs/nfs/nfs4proc.c +0 -0
@@ -4616,6 +4618,7 @@ struct nfs4_session *nfs4_alloc_session(struct=

nfs_client *clp)

Quoted 4 lines

fs/nfs/nfs4proc.c struct nfs4_session *nfs4_alloc_session(struct= · line 4620

Quoted 4 lines
fs/nfs/nfs4proc.c +1 -0
 =A0void nfs4_destroy_session(struct nfs4_session *session)
 =A0{
+ =A0 =A0 set_bit(NFS4CLNT_SESSION_DESTROY, &session->clp->cl_state)=

;

Quoted 3 lines
Quoted 3 lines
fs/nfs/nfs4proc.c +0 -0
 =A0 =A0 =A0 nfs4_proc_destroy_session(session);
 =A0 =A0 =A0 dprintk("%s Destroy backchannel for xprt %p\n",

=A0 =A0 =A0 =A0 =A0 =A0 =A0 __func__, session->clp->cl_rpcclient->cl=

_xprt);

Quoted 5 lines

fs/nfs/nfs4state.c static void nfs4_state_manager(struct nfs_cl=

Quoted 5 lines
fs/nfs/nfs4state.c +0 -0
index 1394dfb..a509b26 100644
@@ -1253,11 +1253,16 @@ static void nfs4_state_manager(struct nfs_cl=

ient *clp)

Quoted 3 lines
Quoted 3 lines
fs/nfs/nfs4state.c +0 -0
 =A0 =A0 =A0 =A0 =A0 =A0 =A0 }

=A0 =A0 =A0 =A0 =A0 =A0 =A0 /* Initialize or reset the session */ =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (test_and_clear_bit(NFS4CLNT_SESSION_=

SETUP, &clp->cl_state)

Quoted 3 lines
Quoted 3 lines

- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0&& nfs4_has_session(clp)) {
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 && nfs4_has_session(clp)) { =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (clp->cl_cons_state =3D=

=3D NFS_CS_SESSION_INITING)

Quoted 1 line
Quoted 1 line

=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 status =3D=

nfs4_initialize_session(clp);

Quoted 2 lines
Quoted 2 lines

- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 else
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 status =3D=

nfs4_reset_session(clp);

Quoted 2 lines
Quoted 2 lines

+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 else {
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (test_b=

it(NFS4CLNT_SESSION_DESTROY,

Quoted 1 line
Quoted 1 line

+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0=

=A0 =A0 =A0 =A0&clp->cl_state))

Quoted 1 line
Quoted 1 line

+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0=

=A0 status =3D -EIO;

Quoted 2 lines
Quoted 2 lines

+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 else
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0=

=A0 status =3D nfs4_reset_session(clp);

Quoted 3 lines
Quoted 3 lines

+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 }
=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (status) { =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 if (stat=

us =3D=3D -NFS4ERR_STALE_CLIENTID)

Quoted 1 line
Quoted 1 line

=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =

=A0 =A0 continue;

Quoted 4 lines

Wait... Why is all this needed?

AFAICS, nfs_destroy_session() is only called by nfs_free_client() onc=

e

Quoted 1 line

the very last reference to the nfs_client has been released. Now sinc=

e

Quoted 1 line

the state manager keeps a reference to the nfs_client all the time wh=

ile

Quoted 3 lines

it is running (see nfs4_run_state_manager()), we _can't_ have a race between it and nfs4_destroy_session().

This is correct very and in the normal case it is not a problem. However, the problems start when the server is down/rebooted/etc and the client tries to umount.
Then, the (SEQUENCE) requests are _already_ queued and when the server comes up or the request is cancelled (ctl-c), we see these race conditions:

a) nfs41_sequence_done() called after destroy_session() that leads to a NULL pointer dereference
b) a BADSESSION reply to a sequence operation triggers a reset_session() at the same time with destroy_session() (called by umount) that leads to another NULL pointer dereference.

I can send you the call traces if you need more info. I understand that these are pretty rare cases, but they shouldn't lead to a crash.

-alexandros

Quoted 5 lines

_______________________________________________
pNFS mailing list
[email protected]
http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs

A message that is not in the archive (sent off-list or not yet received): 5e24e8930910141450h11e677bbr17ebe3441a8742d8-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org

#5 lore raw
Quoted 5 lines · Alexandros Batsakis

a) nfs41_sequence_done() called after destroy_session() that leads to a NULL pointer dereference
b) a BADSESSION reply to a sequence operation triggers a reset_session() at the same time with destroy_session() (called by umount) that leads to another NULL pointer dereference.

This would mean that nfs41_sequence_done is being called _after_ the nfs_client (and hence the session) has been destroyed. That sounds like the real bug that needs to be fixed.

Cheers
Trond

A message that is not in the archive (sent off-list or not yet received): 1255557198.6308.36.camel-rJ7iovZKK19ZJLDQqaL3InhyD016LWXt@public.gmane.org

#7 lore raw
Quoted 10 lines · Trond Myklebust
Quoted 5 lines · Alexandros Batsakis

a) nfs41_sequence_done() called after destroy_session() that leads to a NULL pointer dereference
b) a BADSESSION reply to a sequence operation triggers a reset_session() at the same time with destroy_session() (called by umount) that leads to another NULL pointer dereference.

This would mean that nfs41_sequence_done is being called _after_ the nfs_client (and hence the session) has been destroyed. That sounds like the real bug that needs to be fixed.

Correction: it means that nfs41_sequence_done is being called after the superblock that "owns" those rpc calls has been destroyed. (Which is a bug... :-))

Cheers
Trond

A message that is not in the archive (sent off-list or not yet received): 1255557476.6308.39.camel-rJ7iovZKK19ZJLDQqaL3InhyD016LWXt@public.gmane.org

#9 lore raw
Quoted 3 lines · Trond Myklebust
Quoted 2 lines · Trond Myklebust
Quoted 1 line · Alexandros Batsakis

a) nfs41_sequence_done() called after destroy_session() that leads=

to

Quoted 7 lines
Quoted 7 lines
Quoted 4 lines

a NULL pointer dereference
b) a BADSESSION reply to a sequence operation triggers a reset_session() at the same time with destroy_session() (called by umount) that leads to another NULL pointer dereference.

This would mean that nfs41_sequence_done is being called _after_ the nfs_client (and hence the session) has been destroyed. That sounds l=

ike

Quoted 3 lines
Quoted 1 line

the real bug that needs to be fixed.

Correction: it means that nfs41_sequence_done is being called after t=

he

Quoted 1 line

superblock that "owns" those rpc calls has been destroyed. (Which is =

a

Quoted 2 lines

bug... :-))

Agreed. FWIW and from a conceptual point of view, the patch above is a bit orthogonal to that as it deals with the problem within the session scope. It treats the umount just as a session destroyer that happens to always destroy the super-block in the current one-session-per-client implementation. The latter may change, but the patch will remain relevant (obviously with few adjustments). Anyway, as long as we fix the bug I am happy :)

-alexandros

Quoted 8 lines

Cheers
=A0Trond

_______________________________________________
pNFS mailing list
[email protected]
http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs

A message that is not in the archive (sent off-list or not yet received): 5e24e8930910141509j3fad98afr66e25c465ced1e42-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org

#11 lore raw
Quoted 19 lines · Alexandros Batsakis
Quoted 8 lines · Trond Myklebust
Quoted 7 lines · Trond Myklebust
Quoted 5 lines · Alexandros Batsakis

a) nfs41_sequence_done() called after destroy_session() that leads to a NULL pointer dereference
b) a BADSESSION reply to a sequence operation triggers a reset_session() at the same time with destroy_session() (called by umount) that leads to another NULL pointer dereference.

Quoted 8 lines
Quoted 3 lines

This would mean that nfs41_sequence_done is being called _after_ the nfs_client (and hence the session) has been destroyed. That sounds like the real bug that needs to be fixed.

Correction: it means that nfs41_sequence_done is being called after the superblock that "owns" those rpc calls has been destroyed. (Which is a bug... :-))

Quoted 6 lines

Agreed. FWIW and from a conceptual point of view, the patch above is a bit orthogonal to that as it deals with the problem within the session scope. It treats the umount just as a session destroyer that happens to always destroy the super-block in the current one-session-per-client implementation. The latter may change, but the patch will remain relevant (obviously with few adjustments).

I completely disagree. If your code may end up starting the state manager after the nfs_client is destroyed, then you _will_ corrupt significant swathes of memory.

Quoted 1 line

Anyway, as long as we fix the bug I am happy :)

I don't think you have...

Trond