nfsd: make service created per net

PATCH In review Stanislav Kinsbursky 2 participants 10 messages Dec 6, 2012 – Dec 10, 2012

Subsystems (2)

NFS client NFS server

lore

Share & tools

lore

Stanislav Kinsbursky

author

2012-12-06 11:22 UTC #0 lore raw

cover nfsd: make service created per net

This patch set prepares service control to be able to start separated independent NFSd services in different network namespace environments. Key changes are:
1) Create independent instances of NFSd service (and thus threads pool) for each network namespace.
2) Control generic resources allocation and destruction by usage counter.
3) Cleanup and simplify Nfsd service start and shutdown.
4) Move some more NFSd state parts (like boot_time and "nfsd is up" flag to per-net data).

The following series implements...

---

Stanislav Kinsbursky (8):
nfsd: move per-net startup code to separated function
nfsd: per-net NFSd up flag introduced
nfsd: make NFSd service boot time per-net
nfsd: make NFSd service structure allocated per net
nfsd: introduce helpers for generic resources init and shutdown
nfsd: simplify NFSv4 state init and shutdown
nfsd: replace boolean nfsd_up flag by users counter
nfsd: simplify service shutdown

 fs/nfsd/netns.h     |    9 ++
 fs/nfsd/nfs3xdr.c   |   14 +++-
 fs/nfsd/nfs4proc.c  |   11 ++-
 fs/nfsd/nfs4state.c |   14 ++--
 fs/nfsd/nfsctl.c    |   63 ++++++++++-------
 fs/nfsd/nfsd.h      |   23 +-----
 fs/nfsd/nfssvc.c    |  185 +++++++++++++++++++++++++++++++++------------------
 7 files changed, 195 insertions(+), 124 deletions(-)

Stanislav Kinsbursky

author

2012-12-06 11:22 UTC #1 lore raw

PATCH 1/8 nfsd: move per-net startup code to separated function

NFSd resources are partially per-net and partially globally used. This patch splits resources init and shutdown and moves per-net code to separated functions.
Generic and per-net init and shutdown are called sequentially for a while.

Signed-off-by Stanislav Kinsbursky <[email protected]>

---
 fs/nfsd/nfssvc.c |   48 +++++++++++++++++++++++++++++++++---------------
 1 files changed, 33 insertions(+), 15 deletions(-)

fs/nfsd/nfssvc.c +33 ~~-15~~

index b144658..038348b 100644
@@ -203,6 +203,27 @@ static int nfsd_init_socks(struct net *net)
 
 static bool nfsd_up = false;
 
+static int nfsd_startup_net(struct net *net)
+{
+	int ret;
+
+	ret = nfsd_init_socks(net);
+	if (ret)
+		return ret;
+	ret = lockd_up(net);
+	if (ret)
+		return ret;
+	ret = nfs4_state_start_net(net);
+	if (ret)
+		goto out_lockd;
+
+	return 0;
+
+out_lockd:
+	lockd_down(net);
+	return ret;
+}
+
 static int nfsd_startup(int nrservs, struct net *net)
 {
 	int ret;

@@ -217,31 +238,29 @@ static int nfsd_startup(int nrservs, struct net *net)
 	ret = nfsd_racache_init(2*nrservs);
 	if (ret)
 		return ret;
-	ret = nfsd_init_socks(net);
-	if (ret)
-		goto out_racache;
-	ret = lockd_up(net);
-	if (ret)
-		goto out_racache;
 	ret = nfs4_state_start();
 	if (ret)
-		goto out_lockd;
-
-	ret = nfs4_state_start_net(net);
+		goto out_racache;
+	ret = nfsd_startup_net(net);
 	if (ret)
-		goto out_net_state;
+		goto out_net;
 
 	nfsd_up = true;
 	return 0;
-out_net_state:
+
+out_net:
 	nfs4_state_shutdown();
-out_lockd:
-	lockd_down(net);
 out_racache:
 	nfsd_racache_shutdown();
 	return ret;
 }
 
+static void nfsd_shutdown_net(struct net *net)
+{
+	nfs4_state_shutdown_net(net);
+	lockd_down(net);
+}
+
 static void nfsd_shutdown(struct net *net)
 {
 	/*

@@ -252,9 +271,8 @@ static void nfsd_shutdown(struct net *net)
 	 */
 	if (!nfsd_up)
 		return;
-	nfs4_state_shutdown_net(net);
+	nfsd_shutdown_net(net);
 	nfs4_state_shutdown();
-	lockd_down(net);
 	nfsd_racache_shutdown();
 	nfsd_up = false;
 }

Stanislav Kinsbursky

author

2012-12-06 11:22 UTC #2 lore raw

PATCH 2/8 nfsd: per-net NFSd up flag introduced

In reply to Stanislav Kinsbursky

This patch introduces introduces per-net "nfsd_net_up" boolean flag, which has the same purpose as general "nfsd_up" flag - skip init or shutdown of per-net resources in case of they are inited on shutted down respectively.

Signed-off-by Stanislav Kinsbursky <[email protected]>

---
 fs/nfsd/netns.h  |    2 ++
 fs/nfsd/nfssvc.c |   12 ++++++++++++
 2 files changed, 14 insertions(+), 0 deletions(-)

fs/nfsd/netns.h +2 -0

index bca789d..da8741b 100644
@@ -92,6 +92,8 @@ struct nfsd_net {
 
 	time_t nfsd4_lease;
 	time_t nfsd4_grace;
+
+	bool nfsd_net_up;
 };
 
 extern int nfsd_net_id;

fs/nfsd/nfssvc.c +12 -0

index 038348b..6e17efd 100644
@@ -21,6 +21,7 @@
 #include "nfsd.h"
 #include "cache.h"
 #include "vfs.h"
+#include "netns.h"
 
 #define NFSDDBG_FACILITY	NFSDDBG_SVC

@@ -205,8 +206,12 @@ static bool nfsd_up = false;
 
 static int nfsd_startup_net(struct net *net)
 {
+	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 	int ret;
 
+	if (nn->nfsd_net_up)
+		return 0;
+
 	ret = nfsd_init_socks(net);
 	if (ret)
 		return ret;

@@ -217,6 +222,7 @@ static int nfsd_startup_net(struct net *net)
 	if (ret)
 		goto out_lockd;
 
+	nn->nfsd_net_up = true;
 	return 0;
 
 out_lockd:

@@ -257,8 +263,14 @@ out_racache:
 
 static void nfsd_shutdown_net(struct net *net)
 {
+	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
+
+	if (!nn->nfsd_net_up)
+		return;
+
 	nfs4_state_shutdown_net(net);
 	lockd_down(net);
+	nn->nfsd_net_up = false;
 }
 
 static void nfsd_shutdown(struct net *net)

Stanislav Kinsbursky

author

2012-12-06 11:22 UTC #3 lore raw

PATCH 3/8 nfsd: make NFSd service boot time per-net

In reply to Stanislav Kinsbursky

All is simple: NFSd service can be started in diffent moments of time in different network environments. So, it's "boot time" have to be assigned per net.

Signed-off-by Stanislav Kinsbursky <[email protected]>

---
 fs/nfsd/netns.h    |    5 +++++
 fs/nfsd/nfs3xdr.c  |   14 ++++++++++----
 fs/nfsd/nfs4proc.c |   11 ++++++-----
 fs/nfsd/nfsd.h     |    5 -----
 fs/nfsd/nfssvc.c   |    4 ++--
 5 files changed, 23 insertions(+), 16 deletions(-)

fs/nfsd/netns.h +5 -0

index da8741b..8acfb17 100644
@@ -94,6 +94,11 @@ struct nfsd_net {
 	time_t nfsd4_grace;
 
 	bool nfsd_net_up;
+
+	/*
+	 * Time of server startup
+	 */
+	struct timeval nfssvc_boot;
 };
 
 extern int nfsd_net_id;

fs/nfsd/nfs3xdr.c +10 -4

index 43f46cd..2b8618d 100644
@@ -7,8 +7,10 @@
  */
 
 #include <linux/namei.h>
+#include <linux/sunrpc/svc_xprt.h>
 #include "xdr3.h"
 #include "auth.h"
+#include "netns.h"
 
 #define NFSDDBG_FACILITY		NFSDDBG_XDR

@@ -720,12 +722,14 @@ int
 nfs3svc_encode_writeres(struct svc_rqst *rqstp, __be32 *p,
 					struct nfsd3_writeres *resp)
 {
+	struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+
 	p = encode_wcc_data(rqstp, p, &resp->fh);
 	if (resp->status == 0) {
 		*p++ = htonl(resp->count);
 		*p++ = htonl(resp->committed);
-		*p++ = htonl(nfssvc_boot.tv_sec);
-		*p++ = htonl(nfssvc_boot.tv_usec);
+		*p++ = htonl(nn->nfssvc_boot.tv_sec);
+		*p++ = htonl(nn->nfssvc_boot.tv_usec);
 	}
 	return xdr_ressize_check(rqstp, p);
 }

@@ -1082,11 +1086,13 @@ int
 nfs3svc_encode_commitres(struct svc_rqst *rqstp, __be32 *p,
 					struct nfsd3_commitres *resp)
 {
+	struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+
 	p = encode_wcc_data(rqstp, p, &resp->fh);
 	/* Write verifier */
 	if (resp->status == 0) {
-		*p++ = htonl(nfssvc_boot.tv_sec);
-		*p++ = htonl(nfssvc_boot.tv_usec);
+		*p++ = htonl(nn->nfssvc_boot.tv_sec);
+		*p++ = htonl(nn->nfssvc_boot.tv_usec);
 	}
 	return xdr_ressize_check(rqstp, p);
 }

fs/nfsd/nfs4proc.c +6 -5

index 87d24e5..14406ae 100644
@@ -495,12 +495,13 @@ nfsd4_access(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 			   &access->ac_supported);
 }
 
-static void gen_boot_verifier(nfs4_verifier *verifier)
+static void gen_boot_verifier(nfs4_verifier *verifier, struct net *net)
 {
 	__be32 verf[2];
+	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
-	verf[0] = (__be32)nfssvc_boot.tv_sec;
-	verf[1] = (__be32)nfssvc_boot.tv_usec;
+	verf[0] = (__be32)nn->nfssvc_boot.tv_sec;
+	verf[1] = (__be32)nn->nfssvc_boot.tv_usec;
 	memcpy(verifier->data, verf, sizeof(verifier->data));
 }

@@ -508,7 +509,7 @@ static __be32
 nfsd4_commit(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	     struct nfsd4_commit *commit)
 {
-	gen_boot_verifier(&commit->co_verf);
+	gen_boot_verifier(&commit->co_verf, SVC_NET(rqstp));
 	return nfsd_commit(rqstp, &cstate->current_fh, commit->co_offset,
 			     commit->co_count);
 }

@@ -928,7 +929,7 @@ nfsd4_write(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 
 	cnt = write->wr_buflen;
 	write->wr_how_written = write->wr_stable_how;
-	gen_boot_verifier(&write->wr_verifier);
+	gen_boot_verifier(&write->wr_verifier, SVC_NET(rqstp));
 
 	nvecs = fill_in_write_vector(rqstp->rq_vec, write);
 	WARN_ON_ONCE(nvecs > ARRAY_SIZE(rqstp->rq_vec));

fs/nfsd/nfsd.h +0 -5

index 18f9996..71ba60d 100644
@@ -269,11 +269,6 @@ void		nfsd_lockd_shutdown(void);
 /* Check for dir entries '.' and '..' */
 #define isdotent(n, l)	(l < 3 && n[0] == '.' && (l == 1 || n[1] == '.'))
 
-/*
- * Time of server startup
- */
-extern struct timeval	nfssvc_boot;
-
 #ifdef CONFIG_NFSD_V4
 
 /* before processing a COMPOUND operation, we have to check that there

fs/nfsd/nfssvc.c +2 -2

index 6e17efd..40992cd 100644
@@ -27,7 +27,6 @@
 
 extern struct svc_program	nfsd_program;
 static int			nfsd(void *vrqstp);
-struct timeval			nfssvc_boot;
 
 /*
  * nfsd_mutex protects nfsd_serv -- both the pointer itself and the members

@@ -367,6 +366,7 @@ static int nfsd_get_default_max_blksize(void)
 int nfsd_create_serv(struct net *net)
 {
 	int error;
+	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
 	WARN_ON(!mutex_is_locked(&nfsd_mutex));
 	if (nfsd_serv) {

@@ -388,7 +388,7 @@ int nfsd_create_serv(struct net *net)
 	}
 
 	set_max_drc();
-	do_gettimeofday(&nfssvc_boot);		/* record boot time */
+	do_gettimeofday(&nn->nfssvc_boot);		/* record boot time */
 	return 0;
 }

Stanislav Kinsbursky

author

2012-12-06 11:22 UTC #4 lore raw

PATCH 4/8 nfsd: make NFSd service structure allocated per net

In reply to Stanislav Kinsbursky

This patch makes main step in NFSd containerisation.
There could be different approaches in how to make NFSd able to handle incomming RPC request from different network namespaces.

But there are two main ideas:
1) Share NFSd kthreads betwween all network namespaces.
2) Create separated pool of threads for each namespace.

While first approach looks more flexible, second one is simplier and non-racy. This patch implements second one.
To make it possible to allocate separated pool of threads, we have to make it possible to allocate separated NFSd service structure per net.

Signed-off-by Stanislav Kinsbursky <[email protected]>

---
 fs/nfsd/netns.h     |    2 +
 fs/nfsd/nfs4state.c |   14 +++++---
 fs/nfsd/nfsctl.c    |   63 +++++++++++++++++++++--------------
 fs/nfsd/nfsd.h      |   18 ++--------
 fs/nfsd/nfssvc.c    |   91 ++++++++++++++++++++++++++++++++-------------------
 5 files changed, 110 insertions(+), 78 deletions(-)

fs/nfsd/netns.h +2 -0

index 8acfb17..abfc97c 100644
@@ -99,6 +99,8 @@ struct nfsd_net {
 	 * Time of server startup
 	 */
 	struct timeval nfssvc_boot;
+
+	struct svc_serv *nfsd_serv;
 };
 
 extern int nfsd_net_id;

fs/nfsd/nfs4state.c +9 -5

index b1aa577..39605c1 100644
@@ -743,9 +743,12 @@ out_free:
 	return NULL;
 }
 
-static void init_forechannel_attrs(struct nfsd4_channel_attrs *new, struct nfsd4_channel_attrs *req, int numslots, int slotsize)
+static void init_forechannel_attrs(struct nfsd4_channel_attrs *new,
+				   struct nfsd4_channel_attrs *req,
+				   int numslots, int slotsize,
+				   struct nfsd_net *nn)
 {
-	u32 maxrpc = nfsd_serv->sv_max_mesg;
+	u32 maxrpc = nn->nfsd_serv->sv_max_mesg;
 
 	new->maxreqs = numslots;
 	new->maxresp_cached = min_t(u32, req->maxresp_cached,

@@ -883,7 +886,8 @@ void nfsd4_put_session(struct nfsd4_session *ses)
 	spin_unlock(&nn->client_lock);
 }
 
-static struct nfsd4_session *alloc_session(struct nfsd4_channel_attrs *fchan)
+static struct nfsd4_session *alloc_session(struct nfsd4_channel_attrs *fchan,
+					   struct nfsd_net *nn)
 {
 	struct nfsd4_session *new;
 	int numslots, slotsize;

@@ -904,7 +908,7 @@ static struct nfsd4_session *alloc_session(struct nfsd4_channel_attrs *fchan)
 		nfsd4_put_drc_mem(slotsize, fchan->maxreqs);
 		return NULL;
 	}
-	init_forechannel_attrs(&new->se_fchannel, fchan, numslots, slotsize);
+	init_forechannel_attrs(&new->se_fchannel, fchan, numslots, slotsize, nn);
 	return new;
 }

@@ -1776,7 +1780,7 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 		return nfserr_inval;
 	if (check_forechannel_attrs(cr_ses->fore_channel))
 		return nfserr_toosmall;
-	new = alloc_session(&cr_ses->fore_channel);
+	new = alloc_session(&cr_ses->fore_channel, nn);
 	if (!new)
 		return nfserr_jukebox;
 	status = nfserr_jukebox;

fs/nfsd/nfsctl.c +38 ~~-25~~

206 lines not on this page · raw message

fs/nfsd/nfsd.h +4 ~~-14~~

index 71ba60d..de23db2 100644
@@ -55,7 +55,6 @@ extern struct svc_version	nfsd_version2, nfsd_version3,
 				nfsd_version4;
 extern u32			nfsd_supported_minorversion;
 extern struct mutex		nfsd_mutex;
-extern struct svc_serv		*nfsd_serv;
 extern spinlock_t		nfsd_drc_lock;
 extern unsigned int		nfsd_drc_max_mem;
 extern unsigned int		nfsd_drc_mem_used;

@@ -68,23 +67,14 @@ extern const struct seq_operations nfs_exports_op;
 int		nfsd_svc(int nrservs, struct net *net);
 int		nfsd_dispatch(struct svc_rqst *rqstp, __be32 *statp);
 
-int		nfsd_nrthreads(void);
-int		nfsd_nrpools(void);
-int		nfsd_get_nrthreads(int n, int *);
+int		nfsd_nrthreads(struct net *);
+int		nfsd_nrpools(struct net *);
+int		nfsd_get_nrthreads(int n, int *, struct net *);
 int		nfsd_set_nrthreads(int n, int *, struct net *);
 int		nfsd_pool_stats_open(struct inode *, struct file *);
 int		nfsd_pool_stats_release(struct inode *, struct file *);
 
-static inline void nfsd_destroy(struct net *net)
-{
-	int destroy = (nfsd_serv->sv_nrthreads == 1);
-
-	if (destroy)
-		svc_shutdown_net(nfsd_serv, net);
-	svc_destroy(nfsd_serv);
-	if (destroy)
-		nfsd_serv = NULL;
-}
+void		nfsd_destroy(struct net *net);
 
 #if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL)
 #ifdef CONFIG_NFSD_V2_ACL

fs/nfsd/nfssvc.c +57 ~~-34~~

212 lines not on this page · raw message

Stanislav Kinsbursky

author

2012-12-06 11:22 UTC #5 lore raw

PATCH 5/8 nfsd: introduce helpers for generic resources init and shutdown

In reply to Stanislav Kinsbursky

NFSd have per-net resources and resources, used globally.
Let's move generic resources init and shutdown to separated functions since they are going to be allocated on first NFSd service start and destroyed after last NFSd service shutdown.

Signed-off-by Stanislav Kinsbursky <[email protected]>

---
 fs/nfsd/nfssvc.c |   50 ++++++++++++++++++++++++++++++++++----------------
 1 files changed, 34 insertions(+), 16 deletions(-)

fs/nfsd/nfssvc.c +34 ~~-16~~

index 0e8622a..f9d147f 100644
@@ -206,6 +206,37 @@ static int nfsd_init_socks(struct net *net)
 
 static bool nfsd_up = false;
 
+static int nfsd_startup_generic(int nrservs)
+{
+	int ret;
+
+	if (nfsd_up)
+		return 0;
+
+	/*
+	 * Readahead param cache - will no-op if it already exists.
+	 * (Note therefore results will be suboptimal if number of
+	 * threads is modified after nfsd start.)
+	 */
+	ret = nfsd_racache_init(2*nrservs);
+	if (ret)
+		return ret;
+	ret = nfs4_state_start();
+	if (ret)
+		goto out_racache;
+	return 0;
+
+out_racache:
+	nfsd_racache_shutdown();
+	return ret;
+}
+
+static void nfsd_shutdown_generic(void)
+{
+	nfs4_state_shutdown();
+	nfsd_racache_shutdown();
+}
+
 static int nfsd_startup_net(struct net *net)
 {
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);

@@ -236,19 +267,9 @@ static int nfsd_startup(int nrservs, struct net *net)
 {
 	int ret;
 
-	if (nfsd_up)
-		return 0;
-	/*
-	 * Readahead param cache - will no-op if it already exists.
-	 * (Note therefore results will be suboptimal if number of
-	 * threads is modified after nfsd start.)
-	 */
-	ret = nfsd_racache_init(2*nrservs);
+	ret = nfsd_startup_generic(nrservs);
 	if (ret)
 		return ret;
-	ret = nfs4_state_start();
-	if (ret)
-		goto out_racache;
 	ret = nfsd_startup_net(net);
 	if (ret)
 		goto out_net;

@@ -257,9 +278,7 @@ static int nfsd_startup(int nrservs, struct net *net)
 	return 0;
 
 out_net:
-	nfs4_state_shutdown();
-out_racache:
-	nfsd_racache_shutdown();
+	nfsd_shutdown_generic();
 	return ret;
 }

@@ -286,8 +305,7 @@ static void nfsd_shutdown(struct net *net)
 	if (!nfsd_up)
 		return;
 	nfsd_shutdown_net(net);
-	nfs4_state_shutdown();
-	nfsd_racache_shutdown();
+	nfsd_shutdown_generic();
 	nfsd_up = false;
 }

Stanislav Kinsbursky

author

2012-12-06 11:23 UTC #6 lore raw

PATCH 6/8 nfsd: simplify NFSv4 state init and shutdown

In reply to Stanislav Kinsbursky

This patch moves nfsd_startup_generic() and nfsd_shutdown_generic() calls to nfsd_startup_net() and nfsd_shutdown_net() respectively, which allow to call nfsd_startup_net() instead of nfsd_startup() and makes code looks clearer. It alos modifies nfsd_svc() and nfsd_shutdown() to check for nn->nfsd_net_up variable instead of global nfsd_up.
Last one is now used only generic resources shutdown and currently useless. But it will replaced by NFSd users counter later in this series.

Signed-off-by Stanislav Kinsbursky <[email protected]>

---
 fs/nfsd/nfssvc.c |   44 +++++++++++++++-----------------------------
 1 files changed, 15 insertions(+), 29 deletions(-)

fs/nfsd/nfssvc.c +15 ~~-29~~

index f9d147f..0c87b4e 100644
@@ -235,9 +235,10 @@ static void nfsd_shutdown_generic(void)
 {
 	nfs4_state_shutdown();
 	nfsd_racache_shutdown();
+	nfsd_up = false;
 }
 
-static int nfsd_startup_net(struct net *net)
+static int nfsd_startup_net(int nrservs, struct net *net)
 {
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 	int ret;

@@ -245,39 +246,26 @@ static int nfsd_startup_net(struct net *net)
 	if (nn->nfsd_net_up)
 		return 0;
 
-	ret = nfsd_init_socks(net);
+	ret = nfsd_startup_generic(nrservs);
 	if (ret)
 		return ret;
+	ret = nfsd_init_socks(net);
+	if (ret)
+		goto out_socks;
 	ret = lockd_up(net);
 	if (ret)
-		return ret;
+		goto out_socks;
 	ret = nfs4_state_start_net(net);
 	if (ret)
 		goto out_lockd;
 
 	nn->nfsd_net_up = true;
+	nfsd_up = true;
 	return 0;
 
 out_lockd:
 	lockd_down(net);
-	return ret;
-}
-
-static int nfsd_startup(int nrservs, struct net *net)
-{
-	int ret;
-
-	ret = nfsd_startup_generic(nrservs);
-	if (ret)
-		return ret;
-	ret = nfsd_startup_net(net);
-	if (ret)
-		goto out_net;
-
-	nfsd_up = true;
-	return 0;
-
-out_net:
+out_socks:
 	nfsd_shutdown_generic();
 	return ret;
 }

@@ -286,27 +274,25 @@ static void nfsd_shutdown_net(struct net *net)
 {
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
-	if (!nn->nfsd_net_up)
-		return;
-
 	nfs4_state_shutdown_net(net);
 	lockd_down(net);
 	nn->nfsd_net_up = false;
+	nfsd_shutdown_generic();
 }
 
 static void nfsd_shutdown(struct net *net)
 {
+	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
+
 	/*
 	 * write_ports can create the server without actually starting
 	 * any threads--if we get shut down before any threads are
 	 * started, then nfsd_last_thread will be run before any of this
 	 * other initialization has been done.
 	 */
-	if (!nfsd_up)
+	if (!nn->nfsd_net_up)
 		return;
 	nfsd_shutdown_net(net);
-	nfsd_shutdown_generic();
-	nfsd_up = false;
 }
 
 static void nfsd_last_thread(struct svc_serv *serv, struct net *net)

@@ -528,9 +514,9 @@ nfsd_svc(int nrservs, struct net *net)
 	if (error)
 		goto out;
 
-	nfsd_up_before = nfsd_up;
+	nfsd_up_before = nn->nfsd_net_up;
 
-	error = nfsd_startup(nrservs, net);
+	error = nfsd_startup_net(nrservs, net);
 	if (error)
 		goto out_destroy;
 	error = svc_set_num_threads(nn->nfsd_serv, NULL, nrservs);

Stanislav Kinsbursky

author

2012-12-06 11:23 UTC #7 lore raw

PATCH 7/8 nfsd: replace boolean nfsd_up flag by users counter

In reply to Stanislav Kinsbursky

Since we have generic NFSd resurces, we have to introduce some way how to allocate and destroy those resources on first per-net NFSd start and on last per-net NFSd stop respectively.
This patch replaces global boolean nfsd_up flag (which is unused now) by users counter and use it to determine either we need to allocate generic resources or destroy them.

Signed-off-by Stanislav Kinsbursky <[email protected]>

---
 fs/nfsd/nfssvc.c |    9 +++++----
 1 files changed, 5 insertions(+), 4 deletions(-)

fs/nfsd/nfssvc.c +5 -4

index 0c87b4e..5bb4a33 100644
@@ -204,13 +204,13 @@ static int nfsd_init_socks(struct net *net)
 	return 0;
 }
 
-static bool nfsd_up = false;
+static int nfsd_users = 0;
 
 static int nfsd_startup_generic(int nrservs)
 {
 	int ret;
 
-	if (nfsd_up)
+	if (nfsd_users++)
 		return 0;
 
 	/*

@@ -233,9 +233,11 @@ out_racache:
 
 static void nfsd_shutdown_generic(void)
 {
+	if (--nfsd_users)
+		return;
+
 	nfs4_state_shutdown();
 	nfsd_racache_shutdown();
-	nfsd_up = false;
 }
 
 static int nfsd_startup_net(int nrservs, struct net *net)

@@ -260,7 +262,6 @@ static int nfsd_startup_net(int nrservs, struct net *net)
 		goto out_lockd;
 
 	nn->nfsd_net_up = true;
-	nfsd_up = true;
 	return 0;
 
 out_lockd:

Stanislav Kinsbursky

author

2012-12-06 11:23 UTC #8 lore raw

PATCH 8/8 nfsd: simplify service shutdown

In reply to Stanislav Kinsbursky

Function nfsd_shutdown is called from two places: nfsd_last_thread (when last kernel thread is exiting) and nfsd_svc (in case of kthreads starting error). When calling from nfsd_svc(), we can be sure that per-net resources are allocated, so we don't need to check per-net nfsd_net_up boolean flag. This allows us to remove nfsd_shutdown function at all and move check for per-net nfsd_net_up boolean flag to nfsd_last_thread.

Signed-off-by Stanislav Kinsbursky <[email protected]>

---
 fs/nfsd/nfssvc.c |    9 ++-------
 1 files changed, 2 insertions(+), 7 deletions(-)

fs/nfsd/nfssvc.c +2 -7

index 5bb4a33..2cfd9c6 100644
@@ -281,7 +281,7 @@ static void nfsd_shutdown_net(struct net *net)
 	nfsd_shutdown_generic();
 }
 
-static void nfsd_shutdown(struct net *net)
+static void nfsd_last_thread(struct svc_serv *serv, struct net *net)
 {
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);

@@ -294,11 +294,6 @@ static void nfsd_shutdown(struct net *net)
 	if (!nn->nfsd_net_up)
 		return;
 	nfsd_shutdown_net(net);
-}
-
-static void nfsd_last_thread(struct svc_serv *serv, struct net *net)
-{
-	nfsd_shutdown(net);
 
 	svc_rpcb_cleanup(serv, net);

@@ -530,7 +525,7 @@ nfsd_svc(int nrservs, struct net *net)
 	error = nn->nfsd_serv->sv_nrthreads - 1;
 out_shutdown:
 	if (error < 0 && !nfsd_up_before)
-		nfsd_shutdown(net);
+		nfsd_shutdown_net(net);
 out_destroy:
 	nfsd_destroy(net);		/* Release server */
 out:

J. Bruce Fields

2012-12-10 20:18 UTC #9 lore raw

In reply to Stanislav Kinsbursky

Quoted 9 lines · Stanislav Kinsbursky

This patch set prepares service control to be able to start separated independent NFSd services in different network namespace environments. Key changes are:
1) Create independent instances of NFSd service (and thus threads pool) for each network namespace.
2) Control generic resources allocation and destruction by usage counter.
3) Cleanup and simplify Nfsd service start and shutdown.
4) Move some more NFSd state parts (like boot_time and "nfsd is up" flag to per-net data).

OK, thanks, applying.--b.

Quoted 25 lines

The following series implements...
---
Stanislav Kinsbursky (8):
nfsd: move per-net startup code to separated function
nfsd: per-net NFSd up flag introduced
nfsd: make NFSd service boot time per-net
nfsd: make NFSd service structure allocated per net
nfsd: introduce helpers for generic resources init and shutdown
nfsd: simplify NFSv4 state init and shutdown
nfsd: replace boolean nfsd_up flag by users counter
nfsd: simplify service shutdown
 fs/nfsd/netns.h     |    9 ++
 fs/nfsd/nfs3xdr.c   |   14 +++-
 fs/nfsd/nfs4proc.c  |   11 ++-
 fs/nfsd/nfs4state.c |   14 ++--
 fs/nfsd/nfsctl.c    |   63 ++++++++++-------
 fs/nfsd/nfsd.h      |   23 +-----
 fs/nfsd/nfssvc.c    |  185 +++++++++++++++++++++++++++++++++------------------
 7 files changed, 195 insertions(+), 124 deletions(-)