Skip to contentSkip to conversation

nfsd41: do not expire client while in use by current compound

RFC Applied Benny Halevy 2 participants 12 messages May 3, 2010 – May 4, 2010

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover nfsd41: do not expire client while in use by current compound

Bruce, the following patchset compiles but is not fully tested yet. It is based on top of http://marc.info/?l=linux-nfs&m=127288934008791&w=2
Please review.

Thanks,

Benny

[RFC 1/3] nfsd4: use local variable in nfs4svc_encode_compoundres [RFC 2/3] nfsd4: hold a reference on nfs4_client when it is used by a session [RFC 3/3] nfsd4: do not expire nfs41 clients while in use

#1 lore raw

RFC 1/3 nfsd4: use local variable in nfs4svc_encode_compoundres

'cs' is already computed, re-use it.

---
 fs/nfsd/nfs4xdr.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)
fs/nfsd/nfs4xdr.c +2 -2
index b04583c..19ff5a3 100644
@@ -3311,9 +3311,9 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
 		if (cs->status != nfserr_replay_cache) {
 			nfsd4_store_cache_entry(resp);
 			dprintk("%s: SET SLOT STATE TO AVAILABLE\n", __func__);
-			resp->cstate.slot->sl_inuse = false;
+			cs->slot->sl_inuse = false;
 		}
-		nfsd4_put_session(resp->cstate.session);
+		nfsd4_put_session(cs->session);
 	}
 	return 1;
 }
Signature · 1 line
1.6.3.3
#3 lore raw

RFC 2/3 nfsd4: hold a reference on nfs4_client when it is used by a session

In reply to Benny Halevy

Although expire_client unhashes the session form the session table so no new compounds can find it, there's no refcount to keep the nfs4_client structure around while it's in use and referenced in the compound state via the session structure.

---
 fs/nfsd/nfs4callback.c |    2 +-
 fs/nfsd/nfs4state.c    |    4 +++-
 fs/nfsd/nfs4xdr.c      |    1 +
 fs/nfsd/state.h        |    6 ++++++
 4 files changed, 11 insertions(+), 2 deletions(-)
fs/nfsd/nfs4callback.c +1 -1
index 7e32bd3..fef1dbe 100644
@@ -571,7 +571,7 @@ nfsd4_probe_callback(struct nfs4_client *clp)
 	}
 
 	/* the task holds a reference to the nfs4_client struct */
-	atomic_inc(&clp->cl_count);
+	get_nfs4_client(clp);
 
 	do_probe_callback(clp);
 }
fs/nfsd/nfs4state.c +3 -1
index 6dbcaf1..50b75af 100644
@@ -1458,8 +1458,10 @@ nfsd4_sequence(struct svc_rqst *rqstp,
 
 out:
 	/* Hold a session reference until done processing the compound. */
-	if (cstate->session)
+	if (cstate->session) {
 		nfsd4_get_session(cstate->session);
+		get_nfs4_client(cstate->session->se_client);
+	}
 	spin_unlock(&sessionid_lock);
 	/* Renew the clientid on success and on replay */
 	if (cstate->session) {
fs/nfsd/nfs4xdr.c +1 -0
index 19ff5a3..aed733c 100644
@@ -3313,6 +3313,7 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
 			dprintk("%s: SET SLOT STATE TO AVAILABLE\n", __func__);
 			cs->slot->sl_inuse = false;
 		}
+		put_nfs4_client(cs->session->se_client);
 		nfsd4_put_session(cs->session);
 	}
 	return 1;
fs/nfsd/state.h +6 -0
index fefeae2..e3c002e 100644
@@ -231,6 +231,12 @@ struct nfs4_client {
 						/* wait here for slots */
 };
 
+static inline void
+get_nfs4_client(struct nfs4_client *clp)
+{
+	atomic_inc(&clp->cl_count);
+}
+
 /* struct nfs4_client_reset
  * one per old client. Populates reset_str_hashtbl. Filled from conf_id_hashtbl
  * upon lease reset, or from upcall to state_daemon (to read in state
Signature · 1 line
1.6.3.3
#10 lore raw

RFC 3/3 nfsd4: do not expire nfs41 clients while in use

In reply to Benny Halevy

Add a cl_use_count atomic counter to struct nfs4_client. Hold a use count while the client is in use by a compound that begins with SEQUENCE.
Renew the client when the comound completes.
The laundromat skips clients that have a positive use count. Otherwise, the laundromat marks the client as expired by setting cl_use_count to -1.

Note that we don't want to use the state lock to synchronize with nfsd4_sequence as we want to diminish the state lock scope.
An alternative to using atomic_cmpxchg is to grab the sessionid_lock spin lock while accessing cl_use_count, but that would cause some contention in the laundromat if it needs to lock and unlock it for every client.

---
 fs/nfsd/nfs4state.c |   40 ++++++++++++++++++++++++++++++++++------
 fs/nfsd/nfs4xdr.c   |    2 ++
 fs/nfsd/state.h     |    9 +++++++++
 3 files changed, 45 insertions(+), 6 deletions(-)
fs/nfsd/nfs4state.c +34 -6
index 50b75af..c95ddca 100644
@@ -647,6 +647,14 @@ renew_client(struct nfs4_client *clp)
 	clp->cl_time = get_seconds();
 }
 
+void
+renew_nfs4_client(struct nfs4_client *clp)
+{
+	nfs4_lock_state();
+	renew_client(clp);
+	nfs4_unlock_state();
+}
+
 /* SETCLIENTID and SETCLIENTID_CONFIRM Helper functions */
 static int
 STALE_CLIENTID(clientid_t *clid)
@@ -715,6 +723,24 @@ put_nfs4_client(struct nfs4_client *clp)
 		free_client(clp);
 }
 
+/*
+ * atomically mark client as used, as long as it's not already expired.
+ * return 0 on success, or a negative value if client already expired.
+ */
+int
+use_nfs4_client(struct nfs4_client *clp)
+{
+	int orig;
+
+	do {
+		orig = atomic_read(&clp->cl_use_count);
+		if (orig < 0)
+			return orig;
+	} while (atomic_cmpxchg(&clp->cl_use_count, orig, orig + 1) != orig);
+
+	return 0;
+}
+
 static void
 expire_client(struct nfs4_client *clp)
 {
@@ -840,6 +866,7 @@ static struct nfs4_client *create_client(struct xdr_netobj name, char *recdir,
 
 	memcpy(clp->cl_recdir, recdir, HEXDIR_LEN);
 	atomic_set(&clp->cl_count, 1);
+	atomic_set(&clp->cl_use_count, 0);
 	atomic_set(&clp->cl_cb_conn.cb_set, 0);
 	INIT_LIST_HEAD(&clp->cl_idhash);
 	INIT_LIST_HEAD(&clp->cl_strhash);
@@ -1423,6 +1450,10 @@ nfsd4_sequence(struct svc_rqst *rqstp,
 	if (!session)
 		goto out;
 
+	/* keep the client from expiring */
+	if (use_nfs4_client(cstate->session->se_client) < 0)
+		goto out;
+
 	status = nfserr_badslot;
 	if (seq->slotid >= session->se_fchannel.maxreqs)
 		goto out;
@@ -1463,12 +1494,6 @@ out:
 		get_nfs4_client(cstate->session->se_client);
 	}
 	spin_unlock(&sessionid_lock);
-	/* Renew the clientid on success and on replay */
-	if (cstate->session) {
-		nfs4_lock_state();
-		renew_client(session->se_client);
-		nfs4_unlock_state();
-	}
 	dprintk("%s: return %d\n", __func__, ntohl(status));
 	return status;
 }
@@ -2575,6 +2600,9 @@ nfs4_laundromat(void)
 		nfsd4_end_grace();
 	list_for_each_safe(pos, next, &client_lru) {
 		clp = list_entry(pos, struct nfs4_client, cl_lru);
+		/* skip client if in use (cl_use_count is greater than zero) */
+		if (atomic_cmpxchg(&clp->cl_use_count, 0, -1) > 0)
+			continue;
 		if (time_after((unsigned long)clp->cl_time, (unsigned long)cutoff)) {
 			t = clp->cl_time - cutoff;
 			if (clientid_val > t)
fs/nfsd/nfs4xdr.c +2 -0
index aed733c..0f0b857 100644
@@ -3313,6 +3313,8 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
 			dprintk("%s: SET SLOT STATE TO AVAILABLE\n", __func__);
 			cs->slot->sl_inuse = false;
 		}
+		renew_nfs4_client(cs->session->se_client);
+		unuse_nfs4_client(cs->session->se_client);
 		put_nfs4_client(cs->session->se_client);
 		nfsd4_put_session(cs->session);
 	}
fs/nfsd/state.h +9 -0
index e3c002e..806d9b8 100644
@@ -214,6 +214,7 @@ struct nfs4_client {
 	nfs4_verifier		cl_confirm;	/* generated by server */
 	struct nfs4_cb_conn	cl_cb_conn;     /* callback info */
 	atomic_t		cl_count;	/* ref count */
+	atomic_t		cl_use_count;	/* session use count */
 	u32			cl_firststate;	/* recovery dir creation */
 
 	/* for nfs41 */
@@ -237,6 +238,12 @@ get_nfs4_client(struct nfs4_client *clp)
 	atomic_inc(&clp->cl_count);
 }
 
+static inline void
+unuse_nfs4_client(struct nfs4_client *clp)
+{
+	atomic_dec(&clp->cl_use_count);
+}
+
 /* struct nfs4_client_reset
  * one per old client. Populates reset_str_hashtbl. Filled from conf_id_hashtbl
  * upon lease reset, or from upcall to state_daemon (to read in state
@@ -384,6 +391,8 @@ extern void nfs4_unlock_state(void);
 extern int nfs4_in_grace(void);
 extern __be32 nfs4_check_open_reclaim(clientid_t *clid);
 extern void put_nfs4_client(struct nfs4_client *clp);
+extern int use_nfs4_client(struct nfs4_client *clp);
+extern void renew_nfs4_client(struct nfs4_client *clp);
 extern void nfs4_free_stateowner(struct kref *kref);
 extern int set_callback_cred(void);
 extern void nfsd4_probe_callback(struct nfs4_client *clp);
Signature · 1 line
1.6.3.3
#4 lore raw

Re: [RFC 2/3] nfsd4: hold a reference on nfs4_client when it is used by a session

In reply to Benny Halevy
Quoted 4 lines · Benny Halevy

Although expire_client unhashes the session form the session table so no new compounds can find it, there's no refcount to keep the nfs4_client structure around while it's in use and referenced in the compound state via the session structure.

The code in my for-2.6.35 branch already has the cl_count removed (and doesn't use it for callbacks any more, instead destroying callbacks before the client is destroyed).

So we need to add a new usage count. I'd prefer to call it something different, since it's being used for something different (cl_users?) since it's being used for something different, and I'd rather avoid confusion with the previous one.

--b.

Quoted 70 lines
---
 fs/nfsd/nfs4callback.c |    2 +-
 fs/nfsd/nfs4state.c    |    4 +++-
 fs/nfsd/nfs4xdr.c      |    1 +
 fs/nfsd/state.h        |    6 ++++++
 4 files changed, 11 insertions(+), 2 deletions(-)
fs/nfsd/nfs4callback.c +1 -1
index 7e32bd3..fef1dbe 100644
@@ -571,7 +571,7 @@ nfsd4_probe_callback(struct nfs4_client *clp)
 	}
 
 	/* the task holds a reference to the nfs4_client struct */
-	atomic_inc(&clp->cl_count);
+	get_nfs4_client(clp);
 
 	do_probe_callback(clp);
 }
fs/nfsd/nfs4state.c +3 -1
index 6dbcaf1..50b75af 100644
@@ -1458,8 +1458,10 @@ nfsd4_sequence(struct svc_rqst *rqstp,
 
 out:
 	/* Hold a session reference until done processing the compound. */
-	if (cstate->session)
+	if (cstate->session) {
 		nfsd4_get_session(cstate->session);
+		get_nfs4_client(cstate->session->se_client);
+	}
 	spin_unlock(&sessionid_lock);
 	/* Renew the clientid on success and on replay */
 	if (cstate->session) {
fs/nfsd/nfs4xdr.c +1 -0
index 19ff5a3..aed733c 100644
@@ -3313,6 +3313,7 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
 			dprintk("%s: SET SLOT STATE TO AVAILABLE\n", __func__);
 			cs->slot->sl_inuse = false;
 		}
+		put_nfs4_client(cs->session->se_client);
 		nfsd4_put_session(cs->session);
 	}
 	return 1;
fs/nfsd/state.h +6 -0
index fefeae2..e3c002e 100644
@@ -231,6 +231,12 @@ struct nfs4_client {
 						/* wait here for slots */
 };
 
+static inline void
+get_nfs4_client(struct nfs4_client *clp)
+{
+	atomic_inc(&clp->cl_count);
+}
+
 /* struct nfs4_client_reset
  * one per old client. Populates reset_str_hashtbl. Filled from conf_id_hashtbl
  * upon lease reset, or from upcall to state_daemon (to read in state
Signature · 1 line
1.6.3.3
#2 lore raw

Re: [RFC 1/3] nfsd4: use local variable in nfs4svc_encode_compoundres

In reply to Benny Halevy
Quoted 1 line · Benny Halevy

'cs' is already computed, re-use it.

Thanks, applied.

--b.

Quoted 25 lines
---
 fs/nfsd/nfs4xdr.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)
fs/nfsd/nfs4xdr.c +2 -2
index b04583c..19ff5a3 100644
@@ -3311,9 +3311,9 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
 		if (cs->status != nfserr_replay_cache) {
 			nfsd4_store_cache_entry(resp);
 			dprintk("%s: SET SLOT STATE TO AVAILABLE\n", __func__);
-			resp->cstate.slot->sl_inuse = false;
+			cs->slot->sl_inuse = false;
 		}
-		nfsd4_put_session(resp->cstate.session);
+		nfsd4_put_session(cs->session);
 	}
 	return 1;
 }
Signature · 1 line
1.6.3.3
#5 lore raw

Re: [RFC 2/3] nfsd4: hold a reference on nfs4_client when it is used by a session

In reply to J. Bruce Fields
Quoted 11 lines · J. Bruce Fields
Quoted 4 lines · Benny Halevy

Although expire_client unhashes the session form the session table so no new compounds can find it, there's no refcount to keep the nfs4_client structure around while it's in use and referenced in the compound state via the session structure.

The code in my for-2.6.35 branch already has the cl_count removed (and doesn't use it for callbacks any more, instead destroying callbacks before the client is destroyed).

So we need to add a new usage count.

(Which you do in the next patch, OK!)

--b.

Quoted 79 lines

I'd prefer to call it something
different, since it's being used for something different (cl_users?) since it's being used for something different, and I'd rather avoid confusion with the previous one.

--b.

Quoted 70 lines
---
 fs/nfsd/nfs4callback.c |    2 +-
 fs/nfsd/nfs4state.c    |    4 +++-
 fs/nfsd/nfs4xdr.c      |    1 +
 fs/nfsd/state.h        |    6 ++++++
 4 files changed, 11 insertions(+), 2 deletions(-)
fs/nfsd/nfs4callback.c +1 -1
index 7e32bd3..fef1dbe 100644
@@ -571,7 +571,7 @@ nfsd4_probe_callback(struct nfs4_client *clp)
 	}
 
 	/* the task holds a reference to the nfs4_client struct */
-	atomic_inc(&clp->cl_count);
+	get_nfs4_client(clp);
 
 	do_probe_callback(clp);
 }
fs/nfsd/nfs4state.c +3 -1
index 6dbcaf1..50b75af 100644
@@ -1458,8 +1458,10 @@ nfsd4_sequence(struct svc_rqst *rqstp,
 
 out:
 	/* Hold a session reference until done processing the compound. */
-	if (cstate->session)
+	if (cstate->session) {
 		nfsd4_get_session(cstate->session);
+		get_nfs4_client(cstate->session->se_client);
+	}
 	spin_unlock(&sessionid_lock);
 	/* Renew the clientid on success and on replay */
 	if (cstate->session) {
fs/nfsd/nfs4xdr.c +1 -0
index 19ff5a3..aed733c 100644
@@ -3313,6 +3313,7 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
 			dprintk("%s: SET SLOT STATE TO AVAILABLE\n", __func__);
 			cs->slot->sl_inuse = false;
 		}
+		put_nfs4_client(cs->session->se_client);
 		nfsd4_put_session(cs->session);
 	}
 	return 1;
fs/nfsd/state.h +6 -0
index fefeae2..e3c002e 100644
@@ -231,6 +231,12 @@ struct nfs4_client {
 						/* wait here for slots */
 };
 
+static inline void
+get_nfs4_client(struct nfs4_client *clp)
+{
+	atomic_inc(&clp->cl_count);
+}
+
 /* struct nfs4_client_reset
  * one per old client. Populates reset_str_hashtbl. Filled from conf_id_hashtbl
  * upon lease reset, or from upcall to state_daemon (to read in state
Signature · 1 line
1.6.3.3
#11 lore raw

Re: [RFC 3/3] nfsd4: do not expire nfs41 clients while in use

In reply to Benny Halevy
Quoted 98 lines · Benny Halevy

Add a cl_use_count atomic counter to struct nfs4_client. Hold a use count while the client is in use by a compound that begins with SEQUENCE.
Renew the client when the comound completes.
The laundromat skips clients that have a positive use count. Otherwise, the laundromat marks the client as expired by setting cl_use_count to -1.

Note that we don't want to use the state lock to synchronize with nfsd4_sequence as we want to diminish the state lock scope.
An alternative to using atomic_cmpxchg is to grab the sessionid_lock spin lock while accessing cl_use_count, but that would cause some contention in the laundromat if it needs to lock and unlock it for every client.

---
 fs/nfsd/nfs4state.c |   40 ++++++++++++++++++++++++++++++++++------
 fs/nfsd/nfs4xdr.c   |    2 ++
 fs/nfsd/state.h     |    9 +++++++++
 3 files changed, 45 insertions(+), 6 deletions(-)
fs/nfsd/nfs4state.c +31 -6
index 50b75af..c95ddca 100644
@@ -647,6 +647,14 @@ renew_client(struct nfs4_client *clp)
 	clp->cl_time = get_seconds();
 }
 
+void
+renew_nfs4_client(struct nfs4_client *clp)
+{
+	nfs4_lock_state();
+	renew_client(clp);
+	nfs4_unlock_state();
+}
+
 /* SETCLIENTID and SETCLIENTID_CONFIRM Helper functions */
 static int
 STALE_CLIENTID(clientid_t *clid)
@@ -715,6 +723,24 @@ put_nfs4_client(struct nfs4_client *clp)
 		free_client(clp);
 }
 
+/*
+ * atomically mark client as used, as long as it's not already expired.
+ * return 0 on success, or a negative value if client already expired.
+ */
+int
+use_nfs4_client(struct nfs4_client *clp)
+{
+	int orig;
+
+	do {
+		orig = atomic_read(&clp->cl_use_count);
+		if (orig < 0)
+			return orig;
+	} while (atomic_cmpxchg(&clp->cl_use_count, orig, orig + 1) != orig);
+
+	return 0;
+}
+
 static void
 expire_client(struct nfs4_client *clp)
 {
@@ -840,6 +866,7 @@ static struct nfs4_client *create_client(struct xdr_netobj name, char *recdir,
 
 	memcpy(clp->cl_recdir, recdir, HEXDIR_LEN);
 	atomic_set(&clp->cl_count, 1);
+	atomic_set(&clp->cl_use_count, 0);
 	atomic_set(&clp->cl_cb_conn.cb_set, 0);
 	INIT_LIST_HEAD(&clp->cl_idhash);
 	INIT_LIST_HEAD(&clp->cl_strhash);
@@ -1423,6 +1450,10 @@ nfsd4_sequence(struct svc_rqst *rqstp,
 	if (!session)
 		goto out;
 
+	/* keep the client from expiring */
+	if (use_nfs4_client(cstate->session->se_client) < 0)
+		goto out;
+
 	status = nfserr_badslot;
 	if (seq->slotid >= session->se_fchannel.maxreqs)
 		goto out;
@@ -1463,12 +1494,6 @@ out:
 		get_nfs4_client(cstate->session->se_client);
 	}
 	spin_unlock(&sessionid_lock);
-	/* Renew the clientid on success and on replay */
-	if (cstate->session) {
-		nfs4_lock_state();
-		renew_client(session->se_client);
-		nfs4_unlock_state();
-	}
 	dprintk("%s: return %d\n", __func__, ntohl(status));
 	return status;
 }

fs/nfsd/nfs4state.c nfs4_laundromat(void) · line 2605

Quoted 6 lines
fs/nfsd/nfs4state.c +3 -0
@@ -2575,6 +2600,9 @@ nfs4_laundromat(void)
 		nfsd4_end_grace();
 	list_for_each_safe(pos, next, &client_lru) {
 		clp = list_entry(pos, struct nfs4_client, cl_lru);
+		/* skip client if in use (cl_use_count is greater than zero) */
+		if (atomic_cmpxchg(&clp->cl_use_count, 0, -1) > 0)
+			continue;

I'll move that check after the time comparison
since there's no need for the atomic instruction if the client's time out hasn't expired yet.

Benny

Quoted 50 lines
fs/nfsd/nfs4state.c +0 -0
 		if (time_after((unsigned long)clp->cl_time, (unsigned long)cutoff)) {
 			t = clp->cl_time - cutoff;
 			if (clientid_val > t)
fs/nfsd/nfs4xdr.c +2 -0
index aed733c..0f0b857 100644
@@ -3313,6 +3313,8 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
 			dprintk("%s: SET SLOT STATE TO AVAILABLE\n", __func__);
 			cs->slot->sl_inuse = false;
 		}
+		renew_nfs4_client(cs->session->se_client);
+		unuse_nfs4_client(cs->session->se_client);
 		put_nfs4_client(cs->session->se_client);
 		nfsd4_put_session(cs->session);
 	}
fs/nfsd/state.h +9 -0
index e3c002e..806d9b8 100644
@@ -214,6 +214,7 @@ struct nfs4_client {
 	nfs4_verifier		cl_confirm;	/* generated by server */
 	struct nfs4_cb_conn	cl_cb_conn;     /* callback info */
 	atomic_t		cl_count;	/* ref count */
+	atomic_t		cl_use_count;	/* session use count */
 	u32			cl_firststate;	/* recovery dir creation */
 
 	/* for nfs41 */
@@ -237,6 +238,12 @@ get_nfs4_client(struct nfs4_client *clp)
 	atomic_inc(&clp->cl_count);
 }
 
+static inline void
+unuse_nfs4_client(struct nfs4_client *clp)
+{
+	atomic_dec(&clp->cl_use_count);
+}
+
 /* struct nfs4_client_reset
  * one per old client. Populates reset_str_hashtbl. Filled from conf_id_hashtbl
  * upon lease reset, or from upcall to state_daemon (to read in state
@@ -384,6 +391,8 @@ extern void nfs4_unlock_state(void);
 extern int nfs4_in_grace(void);
 extern __be32 nfs4_check_open_reclaim(clientid_t *clid);
 extern void put_nfs4_client(struct nfs4_client *clp);
+extern int use_nfs4_client(struct nfs4_client *clp);
+extern void renew_nfs4_client(struct nfs4_client *clp);
 extern void nfs4_free_stateowner(struct kref *kref);
 extern int set_callback_cred(void);
 extern void nfsd4_probe_callback(struct nfs4_client *clp);
#6 lore raw

Re: [RFC 2/3] nfsd4: hold a reference on nfs4_client when it is used by a session

In reply to J. Bruce Fields
Quoted 6 lines · J. Bruce Fields
Quoted 5 lines · J. Bruce Fields
Quoted 4 lines · Benny Halevy

Although expire_client unhashes the session form the session table so no new compounds can find it, there's no refcount to keep the nfs4_client structure around while it's in use and referenced in the compound state via the session structure.

Quoted 7 lines
Quoted 5 lines

The code in my for-2.6.35 branch already has the cl_count removed (and doesn't use it for callbacks any more, instead destroying callbacks before the client is destroyed).

So we need to add a new usage count.

(Which you do in the next patch, OK!)

Right :)

This patch fixes another race in which a client can be expired using expire_client(), not from the laundromat path, while it's being referenced by the session since we look up the session while holding just the sessionid lock and not the state lock. Therefore we must take a refcount on the client while inside the sessionid lock.

Looks like the new usage count in your for-2.6.35 world (that I _think_ could just be a kref now) is needed for delegations as well, right?

Benny

Quoted 85 lines

--b.

Quoted 66 lines

I'd prefer to call it something
different, since it's being used for something different (cl_users?) since it's being used for something different, and I'd rather avoid confusion with the previous one.

--b.

Quoted 57 lines
---
 fs/nfsd/nfs4callback.c |    2 +-
 fs/nfsd/nfs4state.c    |    4 +++-
 fs/nfsd/nfs4xdr.c      |    1 +
 fs/nfsd/state.h        |    6 ++++++
 4 files changed, 11 insertions(+), 2 deletions(-)
fs/nfsd/nfs4callback.c +1 -1
index 7e32bd3..fef1dbe 100644
@@ -571,7 +571,7 @@ nfsd4_probe_callback(struct nfs4_client *clp)
 	}
 
 	/* the task holds a reference to the nfs4_client struct */
-	atomic_inc(&clp->cl_count);
+	get_nfs4_client(clp);
 
 	do_probe_callback(clp);
 }
fs/nfsd/nfs4state.c +3 -1
index 6dbcaf1..50b75af 100644
@@ -1458,8 +1458,10 @@ nfsd4_sequence(struct svc_rqst *rqstp,
 
 out:
 	/* Hold a session reference until done processing the compound. */
-	if (cstate->session)
+	if (cstate->session) {
 		nfsd4_get_session(cstate->session);
+		get_nfs4_client(cstate->session->se_client);
+	}
 	spin_unlock(&sessionid_lock);
 	/* Renew the clientid on success and on replay */
 	if (cstate->session) {
fs/nfsd/nfs4xdr.c +1 -0
index 19ff5a3..aed733c 100644
@@ -3313,6 +3313,7 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
 			dprintk("%s: SET SLOT STATE TO AVAILABLE\n", __func__);
 			cs->slot->sl_inuse = false;
 		}
+		put_nfs4_client(cs->session->se_client);
 		nfsd4_put_session(cs->session);
 	}
 	return 1;
fs/nfsd/state.h +0 -0
index fefeae2..e3c002e 100644
@@ -231,6 +231,12 @@ struct nfs4_client {
 						/* wait here for slots */
 };
 
Quoted 12 lines
Quoted 12 lines
fs/nfsd/state.h +6 -0
+static inline void
+get_nfs4_client(struct nfs4_client *clp)
+{
+	atomic_inc(&clp->cl_count);
+}
+
 /* struct nfs4_client_reset
  * one per old client. Populates reset_str_hashtbl. Filled from conf_id_hashtbl
  * upon lease reset, or from upcall to state_daemon (to read in state
Signature · 1 line
1.6.3.3

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#7 lore raw

Re: [RFC 2/3] nfsd4: hold a reference on nfs4_client when it is used by a session

Quoted 17 lines · Benny Halevy
Quoted 6 lines · J. Bruce Fields
Quoted 5 lines · J. Bruce Fields
Quoted 4 lines · Benny Halevy

Although expire_client unhashes the session form the session table so no new compounds can find it, there's no refcount to keep the nfs4_client structure around while it's in use and referenced in the compound state via the session structure.

Quoted 7 lines
Quoted 5 lines

The code in my for-2.6.35 branch already has the cl_count removed (and doesn't use it for callbacks any more, instead destroying callbacks before the client is destroyed).

So we need to add a new usage count.

(Which you do in the next patch, OK!)

Right :)

Quoted 8 lines

This patch fixes another race in which a client can be expired using expire_client(), not from the laundromat path, while it's being referenced by the session since we look up the session while holding just the sessionid lock and not the state lock. Therefore we must take a refcount on the client while inside the sessionid lock.

Looks like the new usage count in your for-2.6.35 world (that I _think_ could just be a kref now) is needed for delegations as well, right?

We should be able to destroy any delegations, recalls, or other callbacks before we destroy the client, so I don't think we need a reference count for those.

--b.

Quoted 89 lines

Benny

Quoted 85 lines

--b.

Quoted 66 lines

I'd prefer to call it something
different, since it's being used for something different (cl_users?) since it's being used for something different, and I'd rather avoid confusion with the previous one.

--b.

Quoted 57 lines
---
 fs/nfsd/nfs4callback.c |    2 +-
 fs/nfsd/nfs4state.c    |    4 +++-
 fs/nfsd/nfs4xdr.c      |    1 +
 fs/nfsd/state.h        |    6 ++++++
 4 files changed, 11 insertions(+), 2 deletions(-)
fs/nfsd/nfs4callback.c +1 -1
index 7e32bd3..fef1dbe 100644
@@ -571,7 +571,7 @@ nfsd4_probe_callback(struct nfs4_client *clp)
 	}
 
 	/* the task holds a reference to the nfs4_client struct */
-	atomic_inc(&clp->cl_count);
+	get_nfs4_client(clp);
 
 	do_probe_callback(clp);
 }
fs/nfsd/nfs4state.c +3 -1
index 6dbcaf1..50b75af 100644
@@ -1458,8 +1458,10 @@ nfsd4_sequence(struct svc_rqst *rqstp,
 
 out:
 	/* Hold a session reference until done processing the compound. */
-	if (cstate->session)
+	if (cstate->session) {
 		nfsd4_get_session(cstate->session);
+		get_nfs4_client(cstate->session->se_client);
+	}
 	spin_unlock(&sessionid_lock);
 	/* Renew the clientid on success and on replay */
 	if (cstate->session) {
fs/nfsd/nfs4xdr.c +1 -0
index 19ff5a3..aed733c 100644
@@ -3313,6 +3313,7 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
 			dprintk("%s: SET SLOT STATE TO AVAILABLE\n", __func__);
 			cs->slot->sl_inuse = false;
 		}
+		put_nfs4_client(cs->session->se_client);
 		nfsd4_put_session(cs->session);
 	}
 	return 1;
fs/nfsd/state.h +0 -0
index fefeae2..e3c002e 100644
@@ -231,6 +231,12 @@ struct nfs4_client {
 						/* wait here for slots */
 };
 
Quoted 12 lines
Quoted 12 lines
fs/nfsd/state.h +6 -0
+static inline void
+get_nfs4_client(struct nfs4_client *clp)
+{
+	atomic_inc(&clp->cl_count);
+}
+
 /* struct nfs4_client_reset
  * one per old client. Populates reset_str_hashtbl. Filled from conf_id_hashtbl
  * upon lease reset, or from upcall to state_daemon (to read in state
Signature · 1 line
1.6.3.3

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#8 lore raw

Re: [RFC 2/3] nfsd4: hold a reference on nfs4_client when it is used by a session

In reply to Benny Halevy
Quoted 11 lines · Benny Halevy
Quoted 10 lines · J. Bruce Fields
Quoted 9 lines · J. Bruce Fields
Quoted 4 lines · Benny Halevy

Although expire_client unhashes the session form the session table so no new compounds can find it, there's no refcount to keep the nfs4_client structure around while it's in use and referenced in the compound state via the session structure.

The code in my for-2.6.35 branch already has the cl_count removed (and doesn't use it for callbacks any more, instead destroying callbacks before the client is destroyed).

Quoted 8 lines
Quoted 3 lines
Quoted 1 line

So we need to add a new usage count.

(Which you do in the next patch, OK!)

Right :)

This patch fixes another race in which a client can be expired using expire_client(), not from the laundromat path,

Ouch, OK, I'd forgotten that case.

--b.

Quoted 96 lines

while it's being referenced
by the session since we look up the session while holding just the sessionid lock and not the state lock. Therefore we must take a refcount on the client while inside the sessionid lock.

Looks like the new usage count in your for-2.6.35 world (that I _think_ could just be a kref now) is needed for delegations as well, right?

Benny

Quoted 85 lines

--b.

Quoted 66 lines

I'd prefer to call it something
different, since it's being used for something different (cl_users?) since it's being used for something different, and I'd rather avoid confusion with the previous one.

--b.

Quoted 57 lines
---
 fs/nfsd/nfs4callback.c |    2 +-
 fs/nfsd/nfs4state.c    |    4 +++-
 fs/nfsd/nfs4xdr.c      |    1 +
 fs/nfsd/state.h        |    6 ++++++
 4 files changed, 11 insertions(+), 2 deletions(-)
fs/nfsd/nfs4callback.c +1 -1
index 7e32bd3..fef1dbe 100644
@@ -571,7 +571,7 @@ nfsd4_probe_callback(struct nfs4_client *clp)
 	}
 
 	/* the task holds a reference to the nfs4_client struct */
-	atomic_inc(&clp->cl_count);
+	get_nfs4_client(clp);
 
 	do_probe_callback(clp);
 }
fs/nfsd/nfs4state.c +3 -1
index 6dbcaf1..50b75af 100644
@@ -1458,8 +1458,10 @@ nfsd4_sequence(struct svc_rqst *rqstp,
 
 out:
 	/* Hold a session reference until done processing the compound. */
-	if (cstate->session)
+	if (cstate->session) {
 		nfsd4_get_session(cstate->session);
+		get_nfs4_client(cstate->session->se_client);
+	}
 	spin_unlock(&sessionid_lock);
 	/* Renew the clientid on success and on replay */
 	if (cstate->session) {
fs/nfsd/nfs4xdr.c +1 -0
index 19ff5a3..aed733c 100644
@@ -3313,6 +3313,7 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
 			dprintk("%s: SET SLOT STATE TO AVAILABLE\n", __func__);
 			cs->slot->sl_inuse = false;
 		}
+		put_nfs4_client(cs->session->se_client);
 		nfsd4_put_session(cs->session);
 	}
 	return 1;
fs/nfsd/state.h +0 -0
index fefeae2..e3c002e 100644
@@ -231,6 +231,12 @@ struct nfs4_client {
 						/* wait here for slots */
 };
 
Quoted 12 lines
Quoted 12 lines
fs/nfsd/state.h +6 -0
+static inline void
+get_nfs4_client(struct nfs4_client *clp)
+{
+	atomic_inc(&clp->cl_count);
+}
+
 /* struct nfs4_client_reset
  * one per old client. Populates reset_str_hashtbl. Filled from conf_id_hashtbl
  * upon lease reset, or from upcall to state_daemon (to read in state
Signature · 1 line
1.6.3.3

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#9 lore raw

Re: [RFC 2/3] nfsd4: hold a reference on nfs4_client when it is used by a session

Quoted 14 lines · J. Bruce Fields
Quoted 13 lines · Benny Halevy
Quoted 12 lines · J. Bruce Fields
Quoted 10 lines · J. Bruce Fields
Quoted 4 lines · Benny Halevy

Although expire_client unhashes the session form the session table so no new compounds can find it, there's no refcount to keep the nfs4_client structure around while it's in use and referenced in the compound state via the session structure.

The code in my for-2.6.35 branch already has the cl_count removed (and doesn't use it for callbacks any more, instead destroying callbacks before the client is destroyed).

So we need to add a new usage count.

Quoted 8 lines
Quoted 6 lines
Quoted 1 line

(Which you do in the next patch, OK!)

Right :)

This patch fixes another race in which a client can be expired using expire_client(), not from the laundromat path,

Ouch, OK, I'd forgotten that case.

I'm working on a new version on top of your for-2.6.35 branch that will solve both cases using the new ref count.

Benny

Quoted 99 lines

--b.

Quoted 96 lines

while it's being referenced
by the session since we look up the session while holding just the sessionid lock and not the state lock. Therefore we must take a refcount on the client while inside the sessionid lock.

Looks like the new usage count in your for-2.6.35 world (that I _think_ could just be a kref now) is needed for delegations as well, right?

Benny

Quoted 85 lines

--b.

Quoted 66 lines

I'd prefer to call it something
different, since it's being used for something different (cl_users?) since it's being used for something different, and I'd rather avoid confusion with the previous one.

--b.

Quoted 57 lines
---
 fs/nfsd/nfs4callback.c |    2 +-
 fs/nfsd/nfs4state.c    |    4 +++-
 fs/nfsd/nfs4xdr.c      |    1 +
 fs/nfsd/state.h        |    6 ++++++
 4 files changed, 11 insertions(+), 2 deletions(-)
fs/nfsd/nfs4callback.c +1 -1
index 7e32bd3..fef1dbe 100644
@@ -571,7 +571,7 @@ nfsd4_probe_callback(struct nfs4_client *clp)
 	}
 
 	/* the task holds a reference to the nfs4_client struct */
-	atomic_inc(&clp->cl_count);
+	get_nfs4_client(clp);
 
 	do_probe_callback(clp);
 }
fs/nfsd/nfs4state.c +3 -1
index 6dbcaf1..50b75af 100644
@@ -1458,8 +1458,10 @@ nfsd4_sequence(struct svc_rqst *rqstp,
 
 out:
 	/* Hold a session reference until done processing the compound. */
-	if (cstate->session)
+	if (cstate->session) {
 		nfsd4_get_session(cstate->session);
+		get_nfs4_client(cstate->session->se_client);
+	}
 	spin_unlock(&sessionid_lock);
 	/* Renew the clientid on success and on replay */
 	if (cstate->session) {
fs/nfsd/nfs4xdr.c +1 -0
index 19ff5a3..aed733c 100644
@@ -3313,6 +3313,7 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
 			dprintk("%s: SET SLOT STATE TO AVAILABLE\n", __func__);
 			cs->slot->sl_inuse = false;
 		}
+		put_nfs4_client(cs->session->se_client);
 		nfsd4_put_session(cs->session);
 	}
 	return 1;
fs/nfsd/state.h +0 -0
index fefeae2..e3c002e 100644
@@ -231,6 +231,12 @@ struct nfs4_client {
 						/* wait here for slots */
 };
 
Quoted 12 lines
Quoted 12 lines
fs/nfsd/state.h +6 -0
+static inline void
+get_nfs4_client(struct nfs4_client *clp)
+{
+	atomic_inc(&clp->cl_count);
+}
+
 /* struct nfs4_client_reset
  * one per old client. Populates reset_str_hashtbl. Filled from conf_id_hashtbl
  * upon lease reset, or from upcall to state_daemon (to read in state
Signature · 1 line
1.6.3.3

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html