Skip to contentSkip to conversation

NFSD: limit the number of v4 clients to 1024 per 1GB of system memory

PATCH In review Dai Ngo 2 participants 5 messages Jul 15, 2022 – Jul 16, 2022

Subsystems (2)
lore
Share & tools
lore

Dai Ngo

author reviewer
#0 lore raw

v3 cover NFSD: limit the number of v4 clients to 1024 per 1GB of system memory

This patch series enforces a limit on the number of v4 clients allowed in the system. With Courteous server support there are potentially a lots courtesy clients exist in the system that use up memory resource preventing them to be used by other components in the system. Also without a limit on the number of clients, the number of clients can grow to a very large number even for system with small memory configuration eventually render the system into an unusable state.

v2:
. move all defines to nfsd.h
. replace unsigned int nfs4_max_client to int
. kick start laundromat in alloc_client when max client reached. . restyle compute of maxreap in nfs4_get_client_reaplist to oneline. . redo enforce of maxreap in nfs4_get_client_reaplist for readability . use bit-wise interger to compute usable memory in nfsd_init_net. . replace NFS4_MAX_CLIENTS_PER_4GB to NFS4_CLIENTS_PER_GB. . use all memory, including high mem, to compute max client.

v3:
. refactoring v4 initialization specific code to a helper in nfs4state.c . fix kernel test robot issue with NFS4_CLIENTS_PER_GB when
CONFIG_NFSD_V4 is not defined by moving v4 specific code to helper nfsd4_init_leases_net in nfs4state.c

---

Dai Ngo (2):
NFSD: refactoring v4 specific code to a helper in nfs4state.c
NFSD: keep track of the number of v4 clients in the system
NFSD: limit the number of v4 clients to 1024 per 1GB of system memory

 fs/nfsd/netns.h     |  3 +++
 fs/nfsd/nfs4state.c | 49 ++++++++++++++++++++++++++++++++++++++++--------
 fs/nfsd/nfsctl.c    |  9 +--------
 fs/nfsd/nfsd.h      |  6 ++++++
 4 files changed, 51 insertions(+), 16 deletions(-)

--
Dai Ngo

author reviewer
#1 lore raw

PATCH v3 1/3 NFSD: refactoring v4 specific code to a helper in nfs4state.c

This patch moves the v4 specific code from nfsd_init_net() to nfsd4_init_leases_net() helper in nfs4state.c

---
 fs/nfsd/nfs4state.c | 12 ++++++++++++
 fs/nfsd/nfsctl.c    |  9 +--------
 fs/nfsd/nfsd.h      |  4 ++++
 3 files changed, 17 insertions(+), 8 deletions(-)
fs/nfsd/nfs4state.c +12 -0
index 9409a0dc1b76..dd00c6ae7501 100644
@@ -4330,6 +4330,18 @@ nfsd4_init_slabs(void)
 	return -ENOMEM;
 }
 
+void nfsd4_init_leases_net(struct nfsd_net *nn)
+{
+	nn->nfsd4_lease = 90;	/* default lease time */
+	nn->nfsd4_grace = 90;
+	nn->somebody_reclaimed = false;
+	nn->track_reclaim_completes = false;
+	nn->clverifier_counter = prandom_u32();
+	nn->clientid_base = prandom_u32();
+	nn->clientid_counter = nn->clientid_base + 1;
+	nn->s2s_cp_cl_id = nn->clientid_counter++;
+}
+
 static void init_nfs4_replay(struct nfs4_replay *rp)
 {
 	rp->rp_status = nfserr_serverfault;
fs/nfsd/nfsctl.c +1 -8
index 0621c2faf242..879b9e4f723b 100644
@@ -1475,14 +1475,7 @@ static __net_init int nfsd_init_net(struct net *net)
 	retval = nfsd_reply_cache_init(nn);
 	if (retval)
 		goto out_drc_error;
-	nn->nfsd4_lease = 90;	/* default lease time */
-	nn->nfsd4_grace = 90;
-	nn->somebody_reclaimed = false;
-	nn->track_reclaim_completes = false;
-	nn->clverifier_counter = prandom_u32();
-	nn->clientid_base = prandom_u32();
-	nn->clientid_counter = nn->clientid_base + 1;
-	nn->s2s_cp_cl_id = nn->clientid_counter++;
+	nfsd4_init_leases_net(nn);
 
 	get_random_bytes(&nn->siphash_key, sizeof(nn->siphash_key));
 	seqlock_init(&nn->writeverf_lock);
fs/nfsd/nfsd.h +4 -0
index 847b482155ae..036015b2dd50 100644
@@ -495,12 +495,16 @@ extern void unregister_cld_notifier(void);
 extern void nfsd4_ssc_init_umount_work(struct nfsd_net *nn);
 #endif
 
+extern void nfsd4_init_leases_net(struct nfsd_net *nn);
+
 #else /* CONFIG_NFSD_V4 */
 static inline int nfsd4_is_junction(struct dentry *dentry)
 {
 	return 0;
 }
 
+static inline void nfsd4_init_leases_net(struct nfsd_net *nn) {};
+
 #define register_cld_notifier() 0
 #define unregister_cld_notifier() do { } while(0)
 
Signature · 1 line
2.9.5
author reviewer
#2 lore raw

PATCH v3 2/3 NFSD: keep track of the number of v4 clients in the system

In reply to Dai Ngo

Add counter nfs4_client_count to keep track of the total number of v4 clients, including courtesy clients, in the system.

---
 fs/nfsd/netns.h     |  2 ++
 fs/nfsd/nfs4state.c | 10 ++++++++--
 2 files changed, 10 insertions(+), 2 deletions(-)
fs/nfsd/netns.h +2 -0
index 1b1a962a1804..ce864f001a3e 100644
@@ -189,6 +189,8 @@ struct nfsd_net {
 	struct nfsd_fcache_disposal *fcache_disposal;
 
 	siphash_key_t		siphash_key;
+
+	atomic_t		nfs4_client_count;
 };
 
 /* Simple check to find out if a given net was properly initialized */
fs/nfsd/nfs4state.c +8 -2
index dd00c6ae7501..8f2b086bf4da 100644
@@ -2053,7 +2053,8 @@ STALE_CLIENTID(clientid_t *clid, struct nfsd_net *nn)
  * This type of memory management is somewhat inefficient, but we use it
  * anyway since SETCLIENTID is not a common operation.
  */
-static struct nfs4_client *alloc_client(struct xdr_netobj name)
+static struct nfs4_client *alloc_client(struct xdr_netobj name,
+				struct nfsd_net *nn)
 {
 	struct nfs4_client *clp;
 	int i;
@@ -2076,6 +2077,7 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name)
 	atomic_set(&clp->cl_rpc_users, 0);
 	clp->cl_cb_state = NFSD4_CB_UNKNOWN;
 	clp->cl_state = NFSD4_ACTIVE;
+	atomic_inc(&nn->nfs4_client_count);
 	atomic_set(&clp->cl_delegs_in_recall, 0);
 	INIT_LIST_HEAD(&clp->cl_idhash);
 	INIT_LIST_HEAD(&clp->cl_openowners);
@@ -2187,6 +2189,7 @@ __destroy_client(struct nfs4_client *clp)
 	struct nfs4_openowner *oo;
 	struct nfs4_delegation *dp;
 	struct list_head reaplist;
+	struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
 
 	INIT_LIST_HEAD(&reaplist);
 	spin_lock(&state_lock);
@@ -2226,6 +2229,7 @@ __destroy_client(struct nfs4_client *clp)
 	nfsd4_shutdown_callback(clp);
 	if (clp->cl_cb_conn.cb_xprt)
 		svc_xprt_put(clp->cl_cb_conn.cb_xprt);
+	atomic_add_unless(&nn->nfs4_client_count, -1, 0);
 	free_client(clp);
 	wake_up_all(&expiry_wq);
 }
@@ -2848,7 +2852,7 @@ static struct nfs4_client *create_client(struct xdr_netobj name,
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 	struct dentry *dentries[ARRAY_SIZE(client_files)];
 
-	clp = alloc_client(name);
+	clp = alloc_client(name, nn);
 	if (clp == NULL)
 		return NULL;
 
@@ -4340,6 +4344,8 @@ void nfsd4_init_leases_net(struct nfsd_net *nn)
 	nn->clientid_base = prandom_u32();
 	nn->clientid_counter = nn->clientid_base + 1;
 	nn->s2s_cp_cl_id = nn->clientid_counter++;
+
+	atomic_set(&nn->nfs4_client_count, 0);
 }
 
 static void init_nfs4_replay(struct nfs4_replay *rp)
Signature · 1 line
2.9.5
author reviewer
#3 lore raw

PATCH v3 3/3 NFSD: limit the number of v4 clients to 1024 per 1GB of system memory

In reply to Dai Ngo

Currently there is no limit on how many v4 clients are supported by the system. This can be a problem in systems with small memory configuration to function properly when a very large number of clients exist that creates memory shortage conditions.

This patch enforces a limit of 1024 NFSv4 clients, including courtesy clients, per 1GB of system memory. When the number of the clients reaches the limit, requests that create new clients are returned with NFS4ERR_DELAY and the laundromat is kicked start to trim old clients. Due to the overhead of the upcall to remove the client record, the maximun number of clients the laundromat removes on each run is limited to 128. This is done to ensure the laundromat can still process the other tasks in a timely manner.

Since there is now a limit of the number of clients, the 24-hr idle time limit of courtesy client is no longer needed and was removed.

---
 fs/nfsd/netns.h     |  1 +
 fs/nfsd/nfs4state.c | 27 +++++++++++++++++++++------
 fs/nfsd/nfsd.h      |  2 ++
 3 files changed, 24 insertions(+), 6 deletions(-)
fs/nfsd/netns.h +1 -0
index ce864f001a3e..ffe17743cc74 100644
@@ -191,6 +191,7 @@ struct nfsd_net {
 	siphash_key_t		siphash_key;
 
 	atomic_t		nfs4_client_count;
+	int			nfs4_max_clients;
 };
 
 /* Simple check to find out if a given net was properly initialized */
fs/nfsd/nfs4state.c +21 -6
index 8f2b086bf4da..28e98ee23d8c 100644
@@ -2059,6 +2059,10 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name,
 	struct nfs4_client *clp;
 	int i;
 
+	if (atomic_read(&nn->nfs4_client_count) >= nn->nfs4_max_clients) {
+		mod_delayed_work(laundry_wq, &nn->laundromat_work, 0);
+		return NULL;
+	}
 	clp = kmem_cache_zalloc(client_slab, GFP_KERNEL);
 	if (clp == NULL)
 		return NULL;
@@ -4336,6 +4340,9 @@ nfsd4_init_slabs(void)
 
 void nfsd4_init_leases_net(struct nfsd_net *nn)
 {
+	u64 max_clients;
+	struct sysinfo si;
+
 	nn->nfsd4_lease = 90;	/* default lease time */
 	nn->nfsd4_grace = 90;
 	nn->somebody_reclaimed = false;
@@ -4346,6 +4353,10 @@ void nfsd4_init_leases_net(struct nfsd_net *nn)
 	nn->s2s_cp_cl_id = nn->clientid_counter++;
 
 	atomic_set(&nn->nfs4_client_count, 0);
+	si_meminfo(&si);
+	max_clients = (u64)si.totalram * si.mem_unit / (1024 * 1024 * 1024);
+	max_clients *= NFS4_CLIENTS_PER_GB;
+	nn->nfs4_max_clients = max_t(int, max_clients, NFS4_CLIENTS_PER_GB);
 }
 
 static void init_nfs4_replay(struct nfs4_replay *rp)
@@ -5812,7 +5823,10 @@ nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist,
 {
 	struct list_head *pos, *next;
 	struct nfs4_client *clp;
+	unsigned int maxreap, reapcnt = 0;
 
+	maxreap = (atomic_read(&nn->nfs4_client_count) >= nn->nfs4_max_clients) ?
+			NFSD_CLIENT_MAX_TRIM_PER_RUN : 0;
 	INIT_LIST_HEAD(reaplist);
 	spin_lock(&nn->client_lock);
 	list_for_each_safe(pos, next, &nn->client_lru) {
@@ -5823,14 +5837,15 @@ nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist,
 			break;
 		if (!atomic_read(&clp->cl_rpc_users))
 			clp->cl_state = NFSD4_COURTESY;
-		if (!client_has_state(clp) ||
-				ktime_get_boottime_seconds() >=
-				(clp->cl_time + NFSD_COURTESY_CLIENT_TIMEOUT))
+		if (!client_has_state(clp))
 			goto exp_client;
-		if (nfs4_anylock_blockers(clp)) {
+		if (!nfs4_anylock_blockers(clp))
+			if (reapcnt >= maxreap)
+				continue;
 exp_client:
-			if (!mark_client_expired_locked(clp))
-				list_add(&clp->cl_lru, reaplist);
+		if (!mark_client_expired_locked(clp)) {
+			list_add(&clp->cl_lru, reaplist);
+			reapcnt++;
 		}
 	}
 	spin_unlock(&nn->client_lock);
fs/nfsd/nfsd.h +2 -0
index 036015b2dd50..a6bfb1b88296 100644
@@ -341,6 +341,8 @@ void		nfsd_lockd_shutdown(void);
 
 #define NFSD_LAUNDROMAT_MINTIMEOUT      1   /* seconds */
 #define	NFSD_COURTESY_CLIENT_TIMEOUT	(24 * 60 * 60)	/* seconds */
+#define	NFSD_CLIENT_MAX_TRIM_PER_RUN	128
+#define	NFS4_CLIENTS_PER_GB		1024
 
 /*
  * The following attributes are currently not supported by the NFSv4 server:
Signature · 1 line
2.9.5
#4 lore raw
In reply to Dai Ngo
Quoted 22 lines · Dai Ngo

This patch series enforces a limit on the number of v4 clients allowed in the system. With Courteous server support there are potentially a lots courtesy clients exist in the system that use up memory resource preventing them to be used by other components in the system. Also without a limit on the number of clients, the number of clients can grow to a very large number even for system with small memory configuration eventually render the system into an unusable state.

v2:
. move all defines to nfsd.h
. replace unsigned int nfs4_max_client to int
. kick start laundromat in alloc_client when max client reached. . restyle compute of maxreap in nfs4_get_client_reaplist to oneline. . redo enforce of maxreap in nfs4_get_client_reaplist for readability . use bit-wise interger to compute usable memory in nfsd_init_net. . replace NFS4_MAX_CLIENTS_PER_4GB to NFS4_CLIENTS_PER_GB. . use all memory, including high mem, to compute max client.

Quoted 5 lines

v3:
. refactoring v4 initialization specific code to a helper in nfs4state.c . fix kernel test robot issue with NFS4_CLIENTS_PER_GB when
CONFIG_NFSD_V4 is not defined by moving v4 specific code
to helper nfsd4_init_leases_net in nfs4state.c

Hey Dai-

I pulled these and applied them with a couple of cosmetic changes. I'm going to run some tests over the weekend before pushing them to for-next.

Quoted 7 lines

---

Dai Ngo (2):
NFSD: refactoring v4 specific code to a helper in nfs4state.c
NFSD: keep track of the number of v4 clients in the system
NFSD: limit the number of v4 clients to 1024 per 1GB of system memory

Quoted 9 lines
fs/nfsd/netns.h     |  3 +++
fs/nfsd/nfs4state.c | 49 ++++++++++++++++++++++++++++++++++++++++--------
fs/nfsd/nfsctl.c    |  9 +--------
fs/nfsd/nfsd.h      |  6 ++++++
4 files changed, 51 insertions(+), 16 deletions(-)

--
Dai Ngo

--
Chuck Lever