Skip to contentSkip to conversation

NFS add session back channel draining

PATCH In review [email protected] 2 participants 3 messages Dec 8, 2010 – Dec 13, 2010 1 message not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 NFS add session back channel draining

From: Andy Adamson <[email protected]>

Currently session draining only drains the fore channel. The back channel processing must also be drained.

Use the back channel highest_slot_used to indicate that a callback is being processed by the callback thread. Move the session complete to be per channel.

When the session is draininig, wait for any current back channel processing to complete and stop all new back channel processing by returning NFS4ERR_DELAY to the back channel client.

Drain the back channel, then the fore channel.

---
 fs/nfs/callback.h         |    1 +
 fs/nfs/callback_proc.c    |    6 ++++++
 fs/nfs/callback_xdr.c     |   17 +++++++++++++++++
 fs/nfs/nfs4proc.c         |   26 +++++++++++++++++++-------
 fs/nfs/nfs4state.c        |   29 ++++++++++++++++++++++-------
 include/linux/nfs_fs_sb.h |    2 +-
 6 files changed, 66 insertions(+), 15 deletions(-)
fs/nfs/callback.h +1 -0
index 85a7cfd..434233e 100644
@@ -131,6 +131,7 @@ extern unsigned nfs4_callback_recallslot(struct cb_recallslotargs *args,
 
 extern __be32 nfs4_callback_getattr(struct cb_getattrargs *args, struct cb_getattrres *res);
 extern __be32 nfs4_callback_recall(struct cb_recallargs *args, void *dummy);
+extern void nfs4_check_drain_bc_complete(struct nfs4_session *ses);
 
 #ifdef CONFIG_NFS_V4
 extern int nfs_callback_up(u32 minorversion, struct rpc_xprt *xprt);
fs/nfs/callback_proc.c +6 -0
index 2950fca..7a976ef 100644
@@ -287,6 +287,12 @@ __be32 nfs4_callback_sequence(struct cb_sequenceargs *args,
 	if (clp == NULL)
 		goto out;
 
+	/* state manager is resetting the session */
+	if (test_bit(NFS4_SESSION_DRAINING, &clp->cl_session->session_state)) {
+		status = NFS4ERR_DELAY;
+		goto out_putclient;
+	}
+
 	status = validate_seqid(&clp->cl_session->bc_slot_table, args);
 	if (status)
 		goto out_putclient;
fs/nfs/callback_xdr.c +17 -0
index 05af212..0a5e32d 100644
@@ -593,6 +593,20 @@ preprocess_nfs41_op(int nop, unsigned int op_nr, struct callback_op **op)
 	return htonl(NFS_OK);
 }
 
+static void nfs4_callback_free_slot(struct nfs4_session *session)
+{
+	struct nfs4_slot_table *tbl = &session->bc_slot_table;
+
+	spin_lock(&tbl->slot_tbl_lock);
+	/*
+	 * Let the state manager know callback processing done.
+	 * A single slot, so highest used slotid is either 0 or -1
+	 */
+	tbl->highest_used_slotid--;
+	nfs4_check_drain_bc_complete(session);
+	spin_unlock(&tbl->slot_tbl_lock);
+}
+
 #else /* CONFIG_NFS_V4_1 */
 
 static __be32
@@ -601,6 +615,9 @@ preprocess_nfs41_op(int nop, unsigned int op_nr, struct callback_op **op)
 	return htonl(NFS4ERR_MINOR_VERS_MISMATCH);
 }
 
+static void nfs4_callback_free_slot(struct nfs4_session *session)
+{
+}
 #endif /* CONFIG_NFS_V4_1 */
 
 static __be32
fs/nfs/nfs4proc.c +19 -7
index 6a653ff..b48f607 100644
@@ -355,9 +355,9 @@ nfs4_free_slot(struct nfs4_slot_table *tbl, struct nfs4_slot *free_slot)
 }
 
 /*
- * Signal state manager thread if session is drained
+ * Signal state manager thread if session fore channel is drained
  */
-static void nfs41_check_drain_session_complete(struct nfs4_session *ses)
+static void nfs4_check_drain_fc_complete(struct nfs4_session *ses)
 {
 	struct rpc_task *task;
 
@@ -371,8 +371,20 @@ static void nfs41_check_drain_session_complete(struct nfs4_session *ses)
 	if (ses->fc_slot_table.highest_used_slotid != -1)
 		return;
 
-	dprintk("%s COMPLETE: Session Drained\n", __func__);
-	complete(&ses->complete);
+	dprintk("%s COMPLETE: Session Fore Channel Drained\n", __func__);
+	complete(&ses->fc_slot_table.complete);
+}
+
+/*
+ * Signal state manager thread if session back channel is drained
+ */
+void nfs4_check_drain_bc_complete(struct nfs4_session *ses)
+{
+	if (!test_bit(NFS4_SESSION_DRAINING, &ses->session_state) ||
+	    ses->bc_slot_table.highest_used_slotid != -1)
+		return;
+	dprintk("%s COMPLETE: Session Back Channel Drained\n", __func__);
+	complete(&ses->bc_slot_table.complete);
 }
 
 static void nfs41_sequence_free_slot(struct nfs4_sequence_res *res)
@@ -389,7 +401,7 @@ static void nfs41_sequence_free_slot(struct nfs4_sequence_res *res)
 
 	spin_lock(&tbl->slot_tbl_lock);
 	nfs4_free_slot(tbl, res->sr_slot);
-	nfs41_check_drain_session_complete(res->sr_session);
+	nfs4_check_drain_fc_complete(res->sr_session);
 	spin_unlock(&tbl->slot_tbl_lock);
 	res->sr_slot = NULL;
 }
@@ -4767,17 +4779,17 @@ struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp)
 	if (!session)
 		return NULL;
 
-	init_completion(&session->complete);
-
 	tbl = &session->fc_slot_table;
 	tbl->highest_used_slotid = -1;
 	spin_lock_init(&tbl->slot_tbl_lock);
 	rpc_init_priority_wait_queue(&tbl->slot_tbl_waitq, "ForeChannel Slot table");
+	init_completion(&tbl->complete);
 
 	tbl = &session->bc_slot_table;
 	tbl->highest_used_slotid = -1;
 	spin_lock_init(&tbl->slot_tbl_lock);
 	rpc_init_wait_queue(&tbl->slot_tbl_waitq, "BackChannel Slot table");
+	init_completion(&tbl->complete);
 
 	session->session_state = 1<<NFS4_SESSION_INITING;
 
fs/nfs/nfs4state.c +22 -7
index f575a31..0082483 100644
@@ -142,6 +142,11 @@ static int nfs41_setup_state_renewal(struct nfs_client *clp)
 	return status;
 }
 
+/*
+ * Back channel returns NFS4ERR_DELAY for new requests when
+ * NFS4_SESSION_DRAINING is set so there is no work to be done when draining
+ * is ended.
+ */
 static void nfs4_end_drain_session(struct nfs_client *clp)
 {
 	struct nfs4_session *ses = clp->cl_session;
@@ -165,21 +170,31 @@ static void nfs4_end_drain_session(struct nfs_client *clp)
 	}
 }
 
-static int nfs4_begin_drain_session(struct nfs_client *clp)
+static int nfs4_wait_on_slot_tbl(struct nfs4_slot_table *tbl)
 {
-	struct nfs4_session *ses = clp->cl_session;
-	struct nfs4_slot_table *tbl = &ses->fc_slot_table;
-
 	spin_lock(&tbl->slot_tbl_lock);
-	set_bit(NFS4_SESSION_DRAINING, &ses->session_state);
 	if (tbl->highest_used_slotid != -1) {
-		INIT_COMPLETION(ses->complete);
+		INIT_COMPLETION(tbl->complete);
 		spin_unlock(&tbl->slot_tbl_lock);
-		return wait_for_completion_interruptible(&ses->complete);
+		return wait_for_completion_interruptible(&tbl->complete);
 	}
 	spin_unlock(&tbl->slot_tbl_lock);
 	return 0;
 }
+	
+static int nfs4_begin_drain_session(struct nfs_client *clp)
+{
+	struct nfs4_session *ses = clp->cl_session;
+	int ret = 0;
+
+	set_bit(NFS4_SESSION_DRAINING, &ses->session_state);
+	/* back channel */
+	ret = nfs4_wait_on_slot_tbl(&ses->bc_slot_table);
+	if (ret)
+		return ret;
+	/* fore channel */
+	return nfs4_wait_on_slot_tbl(&ses->fc_slot_table);
+}
 
 int nfs41_init_clientid(struct nfs_client *clp, struct rpc_cred *cred)
 {
include/linux/nfs_fs_sb.h +1 -1
index 452d964..8ded472 100644
@@ -196,6 +196,7 @@ struct nfs4_slot_table {
 						 * op for dynamic resizing */
 	int		target_max_slots;	/* Set by CB_RECALL_SLOT as
 						 * the new max_slots */
+	struct completion complete;
 };
 
 static inline int slot_idx(struct nfs4_slot_table *tbl, struct nfs4_slot *sp)
@@ -212,7 +213,6 @@ struct nfs4_session {
 	unsigned long			session_state;
 	u32				hash_alg;
 	u32				ssv_len;
-	struct completion		complete;
 
 	/* The fore and back channel */
 	struct nfs4_channel_attrs	fc_attrs;
Signature · 1 line
1.6.6
#1 lore raw
Quoted 6 lines · On Mon, Dec 6, 2010 at 10:19 PM, <[email protected]> wrote:

From: Andy Adamson <[email protected]>

Currently session draining only drains the fore channel. The back channel processing must also be drained.

Use the back channel highest_slot_used to indicate that a callback is=

being

Quoted 1 line

processed by the callback thread. =A0Move the session complete to be =

per channel.

Quoted 2 lines

When the session is draininig, wait for any current back channel proc=

essing

Quoted 1 line

to complete and stop all new back channel processing by returning NFS=

4ERR_DELAY

Quoted 10 lines

to the back channel client.

Drain the back channel, then the fore channel.

---
=A0fs/nfs/callback.h =A0 =A0 =A0 =A0 | =A0 =A01 + =A0fs/nfs/callback_proc.c =A0 =A0| =A0 =A06 ++++++ =A0fs/nfs/callback_xdr.c =A0 =A0 | =A0 17 +++++++++++++++++ =A0fs/nfs/nfs4proc.c =A0 =A0 =A0 =A0 | =A0 26 +++++++++++++++++++----=

---

Quoted 1 line

=A0fs/nfs/nfs4state.c =A0 =A0 =A0 =A0| =A0 29 ++++++++++++++++++++++-=

------

fs/nfs/callback.h extern unsigned nfs4_callback_recallslot(struct c=

Quoted 8 lines

=A0include/linux/nfs_fs_sb.h | =A0 =A02 +- =A06 files changed, 66 insertions(+), 15 deletions(-)

fs/nfs/callback.h +0 -0
index 85a7cfd..434233e 100644
@@ -131,6 +131,7 @@ extern unsigned nfs4_callback_recallslot(struct c=

b_recallslotargs *args,

Quoted 2 lines

=A0extern __be32 nfs4_callback_getattr(struct cb_getattrargs *args, s=

truct cb_getattrres *res);

Quoted 1 line

=A0extern __be32 nfs4_callback_recall(struct cb_recallargs *args, voi=

d *dummy);

Quoted 4 lines

+extern void nfs4_check_drain_bc_complete(struct nfs4_session *ses);

=A0#ifdef CONFIG_NFS_V4
=A0extern int nfs_callback_up(u32 minorversion, struct rpc_xprt *xprt=

);

fs/nfs/callback_proc.c __be32 nfs4_callback_sequence(struct cb_sequence=

Quoted 5 lines
fs/nfs/callback_proc.c +0 -0
index 2950fca..7a976ef 100644
@@ -287,6 +287,12 @@ __be32 nfs4_callback_sequence(struct cb_sequence=

args *args,

fs/nfs/callback_proc.c __be32 nfs4_callback_sequence(struct cb_sequence= · line 291

Quoted 5 lines
fs/nfs/callback_proc.c +2 -0
 =A0 =A0 =A0 =A0if (clp =3D=3D NULL)
 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0goto out;
 
+ =A0 =A0 =A0 /* state manager is resetting the session */
+ =A0 =A0 =A0 if (test_bit(NFS4_SESSION_DRAINING, &clp->cl_session->s=

ession_state)) {

Quoted 5 lines
fs/nfs/callback_proc.c +4 -0
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 status =3D NFS4ERR_DELAY;
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 goto out_putclient;
+ =A0 =A0 =A0 }
+

=A0 =A0 =A0 =A0status =3D validate_seqid(&clp->cl_session->bc_slot_ta=

ble, args);

fs/nfs/callback_xdr.c preprocess_nfs41_op(int nop, unsigned int op_nr,=

Quoted 7 lines

=A0 =A0 =A0 =A0if (status)
=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0goto out_putclient;

fs/nfs/callback_xdr.c +0 -0
index 05af212..0a5e32d 100644
@@ -593,6 +593,20 @@ preprocess_nfs41_op(int nop, unsigned int op_nr,=

struct callback_op **op)

fs/nfs/callback_xdr.c preprocess_nfs41_op(int nop, unsigned int op_nr, struct callback_op **op) · line 596

Quoted 4 lines
fs/nfs/callback_xdr.c +1 -0
 =A0 =A0 =A0 =A0return htonl(NFS_OK);
 =A0}
 
+static void nfs4_callback_free_slot(struct nfs4_session *session)

Am I missing something, or was the call to this function omitted?

=46red

fs/nfs/callback_xdr.c preprocess_nfs41_op(int nop, unsigned int op_nr,= · line 598

Quoted 2 lines
fs/nfs/callback_xdr.c +2 -0
+{
+ =A0 =A0 =A0 struct nfs4_slot_table *tbl =3D &session->bc_slot_table=

;

fs/nfs/callback_xdr.c preprocess_nfs41_op(int nop, unsigned int op_nr,= · line 602

Quoted 4 lines
fs/nfs/callback_xdr.c +4 -0
+
+ =A0 =A0 =A0 spin_lock(&tbl->slot_tbl_lock);
+ =A0 =A0 =A0 /*
+ =A0 =A0 =A0 =A0* Let the state manager know callback processing don=

e.

fs/nfs/callback_xdr.c preprocess_nfs41_op(int nop, unsigned int op_nr,= · line 603

Quoted 1 line
fs/nfs/callback_xdr.c +1 -0
+ =A0 =A0 =A0 =A0* A single slot, so highest used slotid is either 0 =

or -1

fs/nfs/callback_xdr.c preprocess_nfs41_op(int nop, unsigned int op_nr, =

Quoted 10 lines
fs/nfs/callback_xdr.c +6 -0
+ =A0 =A0 =A0 =A0*/
+ =A0 =A0 =A0 tbl->highest_used_slotid--;
+ =A0 =A0 =A0 nfs4_check_drain_bc_complete(session);
+ =A0 =A0 =A0 spin_unlock(&tbl->slot_tbl_lock);
+}
+

=A0#else /* CONFIG_NFS_V4_1 */

=A0static __be32

fs/nfs/callback_xdr.c +0 -0
@@ -601,6 +615,9 @@ preprocess_nfs41_op(int nop, unsigned int op_nr, =

struct callback_op **op)

fs/nfs/nfs4proc.c nfs4_free_slot(struct nfs4_slot_table *tbl, struc=

Quoted 14 lines
fs/nfs/callback_xdr.c +3 -0
 =A0 =A0 =A0 =A0return htonl(NFS4ERR_MINOR_VERS_MISMATCH);
 =A0}
 
+static void nfs4_callback_free_slot(struct nfs4_session *session)
+{
+}

=A0#endif /* CONFIG_NFS_V4_1 */

=A0static __be32

fs/nfs/nfs4proc.c +0 -0
index 6a653ff..b48f607 100644
@@ -355,9 +355,9 @@ nfs4_free_slot(struct nfs4_slot_table *tbl, struc=

t nfs4_slot *free_slot)

Quoted 7 lines
fs/nfs/nfs4proc.c +0 -0
 =A0}
 

=A0/*
- * Signal state manager thread if session is drained
+ * Signal state manager thread if session fore channel is drained =A0*/
-static void nfs41_check_drain_session_complete(struct nfs4_session *=

ses)

fs/nfs/nfs4proc.c static void nfs41_check_drain_session_complete(s=

Quoted 5 lines

+static void nfs4_check_drain_fc_complete(struct nfs4_session *ses) =A0{
=A0 =A0 =A0 =A0struct rpc_task *task;

fs/nfs/nfs4proc.c +0 -0
@@ -371,8 +371,20 @@ static void nfs41_check_drain_session_complete(s=

truct nfs4_session *ses)

fs/nfs/nfs4proc.c static void nfs41_check_drain_session_complete(s= · line 374

Quoted 6 lines
fs/nfs/nfs4proc.c +1 -2
 =A0 =A0 =A0 =A0if (ses->fc_slot_table.highest_used_slotid !=3D -1)
 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0return;
 
- =A0 =A0 =A0 dprintk("%s COMPLETE: Session Drained\n", __func__);
- =A0 =A0 =A0 complete(&ses->complete);
+ =A0 =A0 =A0 dprintk("%s COMPLETE: Session Fore Channel Drained\n", =

__func__);

fs/nfs/nfs4proc.c static void nfs41_check_drain_session_complete(struct nfs4_session *ses)

Quoted 9 lines
fs/nfs/nfs4proc.c +9 -0
+ =A0 =A0 =A0 complete(&ses->fc_slot_table.complete);
+}
+
+/*
+ * Signal state manager thread if session back channel is drained
+ */
+void nfs4_check_drain_bc_complete(struct nfs4_session *ses)
+{
+ =A0 =A0 =A0 if (!test_bit(NFS4_SESSION_DRAINING, &ses->session_stat=

e) ||

fs/nfs/nfs4proc.c static void nfs41_check_drain_session_complete(s= · line 386

Quoted 3 lines
fs/nfs/nfs4proc.c +3 -0
+ =A0 =A0 =A0 =A0 =A0 ses->bc_slot_table.highest_used_slotid !=3D -1)
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 return;
+ =A0 =A0 =A0 dprintk("%s COMPLETE: Session Back Channel Drained\n", =

__func__);

Quoted 4 lines
fs/nfs/nfs4proc.c +1 -0
+ =A0 =A0 =A0 complete(&ses->bc_slot_table.complete);
 =A0}
 

=A0static void nfs41_sequence_free_slot(struct nfs4_sequence_res *res=

)

fs/nfs/nfs4proc.c static void nfs41_sequence_free_slot(struct nfs4_=

Quoted 1 line
fs/nfs/nfs4proc.c +0 -0
@@ -389,7 +401,7 @@ static void nfs41_sequence_free_slot(struct nfs4_=

sequence_res *res)

fs/nfs/nfs4proc.c struct nfs4_session *nfs4_alloc_session(struc=

Quoted 9 lines
fs/nfs/nfs4proc.c +1 -1
 =A0 =A0 =A0 =A0spin_lock(&tbl->slot_tbl_lock);
 =A0 =A0 =A0 =A0nfs4_free_slot(tbl, res->sr_slot);
- =A0 =A0 =A0 nfs41_check_drain_session_complete(res->sr_session);
+ =A0 =A0 =A0 nfs4_check_drain_fc_complete(res->sr_session);
 =A0 =A0 =A0 =A0spin_unlock(&tbl->slot_tbl_lock);
 =A0 =A0 =A0 =A0res->sr_slot =3D NULL;
 =A0}
@@ -4767,17 +4779,17 @@ struct nfs4_session *nfs4_alloc_session(struc=

t nfs_client *clp)

Quoted 9 lines
fs/nfs/nfs4proc.c +0 -2
 =A0 =A0 =A0 =A0if (!session)
 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0return NULL;
 
- =A0 =A0 =A0 init_completion(&session->complete);
-
 =A0 =A0 =A0 =A0tbl =3D &session->fc_slot_table;
 =A0 =A0 =A0 =A0tbl->highest_used_slotid =3D -1;
 =A0 =A0 =A0 =A0spin_lock_init(&tbl->slot_tbl_lock);

=A0 =A0 =A0 =A0rpc_init_priority_wait_queue(&tbl->slot_tbl_waitq, "Fo=

reChannel Slot table");

Quoted 6 lines

+ =A0 =A0 =A0 init_completion(&tbl->complete);

=A0 =A0 =A0 =A0tbl =3D &session->bc_slot_table;
=A0 =A0 =A0 =A0tbl->highest_used_slotid =3D -1;
=A0 =A0 =A0 =A0spin_lock_init(&tbl->slot_tbl_lock);
=A0 =A0 =A0 =A0rpc_init_wait_queue(&tbl->slot_tbl_waitq, "BackChannel=

Slot table");

fs/nfs/nfs4state.c static int nfs41_setup_state_renewal(struct nfs_=

Quoted 9 lines

+ =A0 =A0 =A0 init_completion(&tbl->complete);

=A0 =A0 =A0 =A0session->session_state =3D 1<<NFS4_SESSION_INITING;

fs/nfs/nfs4state.c +0 -0
index f575a31..0082483 100644
@@ -142,6 +142,11 @@ static int nfs41_setup_state_renewal(struct nfs_=

client *clp)

fs/nfs/nfs4state.c static int nfs41_setup_state_renewal(struct nfs_client *clp)

Quoted 6 lines
fs/nfs/nfs4state.c +3 -0
 =A0 =A0 =A0 =A0return status;
 =A0}
 
+/*
+ * Back channel returns NFS4ERR_DELAY for new requests when
+ * NFS4_SESSION_DRAINING is set so there is no work to be done when =

draining

fs/nfs/nfs4state.c static void nfs4_end_drain_session(struct nfs_c=

Quoted 6 lines
fs/nfs/nfs4state.c +2 -0
+ * is ended.
+ */
 =A0static void nfs4_end_drain_session(struct nfs_client *clp)
 =A0{
 =A0 =A0 =A0 =A0struct nfs4_session *ses =3D clp->cl_session;
@@ -165,21 +170,31 @@ static void nfs4_end_drain_session(struct nfs_c=

lient *clp)

Quoted 6 lines
fs/nfs/nfs4state.c +1 -1
 =A0 =A0 =A0 =A0}
 =A0}
 
-static int nfs4_begin_drain_session(struct nfs_client *clp)
+static int nfs4_wait_on_slot_tbl(struct nfs4_slot_table *tbl)
 =A0{

fs/nfs/nfs4state.c static void nfs4_end_drain_session(struct nfs_c= · line 178

Quoted 10 lines
fs/nfs/nfs4state.c +1 -6
- =A0 =A0 =A0 struct nfs4_session *ses =3D clp->cl_session;
- =A0 =A0 =A0 struct nfs4_slot_table *tbl =3D &ses->fc_slot_table;
-
 =A0 =A0 =A0 =A0spin_lock(&tbl->slot_tbl_lock);
- =A0 =A0 =A0 set_bit(NFS4_SESSION_DRAINING, &ses->session_state);
 =A0 =A0 =A0 =A0if (tbl->highest_used_slotid !=3D -1) {
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 INIT_COMPLETION(ses->complete);
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 INIT_COMPLETION(tbl->complete);
 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0spin_unlock(&tbl->slot_tbl_lock);
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 return wait_for_completion_interruptibl=

e(&ses->complete);

fs/nfs/nfs4state.c static void nfs4_end_drain_session(struct nfs_c= · line 179

Quoted 1 line
fs/nfs/nfs4state.c +1 -0
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 return wait_for_completion_interruptibl=

e(&tbl->complete);

Quoted 19 lines
fs/nfs/nfs4state.c +14 -0
 =A0 =A0 =A0 =A0}
 =A0 =A0 =A0 =A0spin_unlock(&tbl->slot_tbl_lock);
 =A0 =A0 =A0 =A0return 0;
 =A0}
+
+static int nfs4_begin_drain_session(struct nfs_client *clp)
+{
+ =A0 =A0 =A0 struct nfs4_session *ses =3D clp->cl_session;
+ =A0 =A0 =A0 int ret =3D 0;
+
+ =A0 =A0 =A0 set_bit(NFS4_SESSION_DRAINING, &ses->session_state);
+ =A0 =A0 =A0 /* back channel */
+ =A0 =A0 =A0 ret =3D nfs4_wait_on_slot_tbl(&ses->bc_slot_table);
+ =A0 =A0 =A0 if (ret)
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 return ret;
+ =A0 =A0 =A0 /* fore channel */
+ =A0 =A0 =A0 return nfs4_wait_on_slot_tbl(&ses->fc_slot_table);
+}
 
Quoted 1 line

=A0int nfs41_init_clientid(struct nfs_client *clp, struct rpc_cred *c=

red)

Quoted 7 lines

=A0{

include/linux/nfs_fs_sb.h +0 -0
index 452d964..8ded472 100644
@@ -196,6 +196,7 @@ struct nfs4_slot_table {

=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0=

=A0 =A0 =A0 =A0 =A0 =A0 * op for dynamic resizing */

Quoted 1 line

=A0 =A0 =A0 =A0int =A0 =A0 =A0 =A0 =A0 =A0 target_max_slots; =A0 =A0 =

=A0 /* Set by CB_RECALL_SLOT as

Quoted 1 line

=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0=

=A0 =A0 =A0 =A0 =A0 =A0 * the new max_slots */

Quoted 4 lines

+ =A0 =A0 =A0 struct completion complete;
=A0};

=A0static inline int slot_idx(struct nfs4_slot_table *tbl, struct nfs=

4_slot *sp)

Quoted 2 lines
include/linux/nfs_fs_sb.h +0 -0
@@ -212,7 +213,6 @@ struct nfs4_session {

=A0 =A0 =A0 =A0unsigned long =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 sess=

ion_state;

Quoted 1 line

=A0 =A0 =A0 =A0u32 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0=

=A0 hash_alg;

Quoted 1 line

=A0 =A0 =A0 =A0u32 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0=

=A0 ssv_len;

Quoted 9 lines

- =A0 =A0 =A0 struct completion =A0 =A0 =A0 =A0 =A0 =A0 =A0 complete;

=A0 =A0 =A0 =A0/* The fore and back channel */
=A0 =A0 =A0 =A0struct nfs4_channel_attrs =A0 =A0 =A0 fc_attrs; --
1.6.6

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" =

in

Quoted 3 lines

the body of a message to [email protected]
More majordomo info at =A0http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): AANLkTi=9ssPfm+DnV76piakHOzq8+BpPTSfyYGEyAPNJ-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org

#3 lore raw
Quoted 62 lines · Fred Isaman
Quoted 61 lines · On Mon, Dec 6, 2010 at 10:19 PM, <[email protected]> wrote:

From: Andy Adamson <[email protected]>

Currently session draining only drains the fore channel. The back channel processing must also be drained.

Use the back channel highest_slot_used to indicate that a callback is being processed by the callback thread. Move the session complete to be per channel.

When the session is draininig, wait for any current back channel processing to complete and stop all new back channel processing by returning NFS4ERR_DELAY to the back channel client.

Drain the back channel, then the fore channel.

---
 fs/nfs/callback.h         |    1 +
 fs/nfs/callback_proc.c    |    6 ++++++
 fs/nfs/callback_xdr.c     |   17 +++++++++++++++++
 fs/nfs/nfs4proc.c         |   26 +++++++++++++++++++-------
 fs/nfs/nfs4state.c        |   29 ++++++++++++++++++++++-------
 include/linux/nfs_fs_sb.h |    2 +-
 6 files changed, 66 insertions(+), 15 deletions(-)
fs/nfs/callback.h +1 -0
index 85a7cfd..434233e 100644
@@ -131,6 +131,7 @@ extern unsigned nfs4_callback_recallslot(struct cb_recallslotargs *args,
 
 extern __be32 nfs4_callback_getattr(struct cb_getattrargs *args, struct cb_getattrres *res);
 extern __be32 nfs4_callback_recall(struct cb_recallargs *args, void *dummy);
+extern void nfs4_check_drain_bc_complete(struct nfs4_session *ses);
 
 #ifdef CONFIG_NFS_V4
 extern int nfs_callback_up(u32 minorversion, struct rpc_xprt *xprt);
fs/nfs/callback_proc.c +6 -0
index 2950fca..7a976ef 100644
@@ -287,6 +287,12 @@ __be32 nfs4_callback_sequence(struct cb_sequenceargs *args,
       if (clp == NULL)
               goto out;
 
+       /* state manager is resetting the session */
+       if (test_bit(NFS4_SESSION_DRAINING, &clp->cl_session->session_state)) {
+               status = NFS4ERR_DELAY;
+               goto out_putclient;
+       }
+
       status = validate_seqid(&clp->cl_session->bc_slot_table, args);
       if (status)
               goto out_putclient;
fs/nfs/callback_xdr.c +0 -0
index 05af212..0a5e32d 100644
@@ -593,6 +593,20 @@ preprocess_nfs41_op(int nop, unsigned int op_nr, struct callback_op **op)
       return htonl(NFS_OK);
 }
 
Quoted 4 lines

fs/nfs/callback_xdr.c preprocess_nfs41_op(int nop, unsigned int op_nr, struct callback_op **op) · line 596

Quoted 1 line
fs/nfs/callback_xdr.c +1 -0
+static void nfs4_callback_free_slot(struct nfs4_session *session)

Am I missing something, or was the call to this function omitted?

You're right, I omitted it - should be in nfs4_callback_compound. I'll check all these patches again, and resend to Trond who I suspect lost them in the flurry of patches...

-->Andy

Quoted 180 lines

Fred

Quoted 177 lines
fs/nfs/callback_xdr.c +16 -0
+{
+       struct nfs4_slot_table *tbl = &session->bc_slot_table;
+
+       spin_lock(&tbl->slot_tbl_lock);
+       /*
+        * Let the state manager know callback processing done.
+        * A single slot, so highest used slotid is either 0 or -1
+        */
+       tbl->highest_used_slotid--;
+       nfs4_check_drain_bc_complete(session);
+       spin_unlock(&tbl->slot_tbl_lock);
+}
+
 #else /* CONFIG_NFS_V4_1 */
 
 static __be32
@@ -601,6 +615,9 @@ preprocess_nfs41_op(int nop, unsigned int op_nr, struct callback_op **op)
       return htonl(NFS4ERR_MINOR_VERS_MISMATCH);
 }
 
+static void nfs4_callback_free_slot(struct nfs4_session *session)
+{
+}
 #endif /* CONFIG_NFS_V4_1 */
 
 static __be32
fs/nfs/nfs4proc.c +19 -7
index 6a653ff..b48f607 100644
@@ -355,9 +355,9 @@ nfs4_free_slot(struct nfs4_slot_table *tbl, struct nfs4_slot *free_slot)
 }
 
 /*
- * Signal state manager thread if session is drained
+ * Signal state manager thread if session fore channel is drained
 */
-static void nfs41_check_drain_session_complete(struct nfs4_session *ses)
+static void nfs4_check_drain_fc_complete(struct nfs4_session *ses)
 {
       struct rpc_task *task;
 
@@ -371,8 +371,20 @@ static void nfs41_check_drain_session_complete(struct nfs4_session *ses)
       if (ses->fc_slot_table.highest_used_slotid != -1)
               return;
 
-       dprintk("%s COMPLETE: Session Drained\n", __func__);
-       complete(&ses->complete);
+       dprintk("%s COMPLETE: Session Fore Channel Drained\n", __func__);
+       complete(&ses->fc_slot_table.complete);
+}
+
+/*
+ * Signal state manager thread if session back channel is drained
+ */
+void nfs4_check_drain_bc_complete(struct nfs4_session *ses)
+{
+       if (!test_bit(NFS4_SESSION_DRAINING, &ses->session_state) ||
+           ses->bc_slot_table.highest_used_slotid != -1)
+               return;
+       dprintk("%s COMPLETE: Session Back Channel Drained\n", __func__);
+       complete(&ses->bc_slot_table.complete);
 }
 
 static void nfs41_sequence_free_slot(struct nfs4_sequence_res *res)
@@ -389,7 +401,7 @@ static void nfs41_sequence_free_slot(struct nfs4_sequence_res *res)
 
       spin_lock(&tbl->slot_tbl_lock);
       nfs4_free_slot(tbl, res->sr_slot);
-       nfs41_check_drain_session_complete(res->sr_session);
+       nfs4_check_drain_fc_complete(res->sr_session);
       spin_unlock(&tbl->slot_tbl_lock);
       res->sr_slot = NULL;
 }
@@ -4767,17 +4779,17 @@ struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp)
       if (!session)
               return NULL;
 
-       init_completion(&session->complete);
-
       tbl = &session->fc_slot_table;
       tbl->highest_used_slotid = -1;
       spin_lock_init(&tbl->slot_tbl_lock);
       rpc_init_priority_wait_queue(&tbl->slot_tbl_waitq, "ForeChannel Slot table");
+       init_completion(&tbl->complete);
 
       tbl = &session->bc_slot_table;
       tbl->highest_used_slotid = -1;
       spin_lock_init(&tbl->slot_tbl_lock);
       rpc_init_wait_queue(&tbl->slot_tbl_waitq, "BackChannel Slot table");
+       init_completion(&tbl->complete);
 
       session->session_state = 1<<NFS4_SESSION_INITING;
 
fs/nfs/nfs4state.c +22 -7
index f575a31..0082483 100644
@@ -142,6 +142,11 @@ static int nfs41_setup_state_renewal(struct nfs_client *clp)
       return status;
 }
 
+/*
+ * Back channel returns NFS4ERR_DELAY for new requests when
+ * NFS4_SESSION_DRAINING is set so there is no work to be done when draining
+ * is ended.
+ */
 static void nfs4_end_drain_session(struct nfs_client *clp)
 {
       struct nfs4_session *ses = clp->cl_session;
@@ -165,21 +170,31 @@ static void nfs4_end_drain_session(struct nfs_client *clp)
       }
 }
 
-static int nfs4_begin_drain_session(struct nfs_client *clp)
+static int nfs4_wait_on_slot_tbl(struct nfs4_slot_table *tbl)
 {
-       struct nfs4_session *ses = clp->cl_session;
-       struct nfs4_slot_table *tbl = &ses->fc_slot_table;
-
       spin_lock(&tbl->slot_tbl_lock);
-       set_bit(NFS4_SESSION_DRAINING, &ses->session_state);
       if (tbl->highest_used_slotid != -1) {
-               INIT_COMPLETION(ses->complete);
+               INIT_COMPLETION(tbl->complete);
               spin_unlock(&tbl->slot_tbl_lock);
-               return wait_for_completion_interruptible(&ses->complete);
+               return wait_for_completion_interruptible(&tbl->complete);
       }
       spin_unlock(&tbl->slot_tbl_lock);
       return 0;
 }
+
+static int nfs4_begin_drain_session(struct nfs_client *clp)
+{
+       struct nfs4_session *ses = clp->cl_session;
+       int ret = 0;
+
+       set_bit(NFS4_SESSION_DRAINING, &ses->session_state);
+       /* back channel */
+       ret = nfs4_wait_on_slot_tbl(&ses->bc_slot_table);
+       if (ret)
+               return ret;
+       /* fore channel */
+       return nfs4_wait_on_slot_tbl(&ses->fc_slot_table);
+}
 
 int nfs41_init_clientid(struct nfs_client *clp, struct rpc_cred *cred)
 {
include/linux/nfs_fs_sb.h +1 -2
index 452d964..8ded472 100644
@@ -196,6 +196,7 @@ struct nfs4_slot_table {
                                                * op for dynamic resizing */
       int             target_max_slots;       /* Set by CB_RECALL_SLOT as
                                                * the new max_slots */
+       struct completion complete;
 };
 
 static inline int slot_idx(struct nfs4_slot_table *tbl, struct nfs4_slot *sp)
@@ -212,7 +213,6 @@ struct nfs4_session {
       unsigned long                   session_state;
       u32                             hash_alg;
       u32                             ssv_len;
-       struct completion               complete;
 
       /* The fore and back channel */
       struct nfs4_channel_attrs       fc_attrs;
--

1.6.6

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html