Skip to contentSkip to conversation

nfsd: more NFSv4 state containerization

PATCH In review Stanislav Kinsbursky 2 participants 7 messages Nov 26, 2012 – Nov 27, 2012

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover nfsd: more NFSv4 state containerization

This patch set makes NFSv4 state created and destroyed per net and thus completes it's containerization (the only exceprtion is client_mutex, but this one a hard nut to crack).

Note: patch set depend on previously sent patch named "nfsd4: remove state lock from nfs4_state_shutdown".

The following series implements...

---

Stanislav Kinsbursky (5):
nfsd: make client_lock per net
nfsd: make delegations shutdown network namespace aware
nfsd: cleanup NFSd state shutdown a bit
nfsd: cleanup NFSd state start a bit
nfsd: call state init and shutdown twice

 fs/nfsd/netns.h     |    3 +
 fs/nfsd/nfs4state.c |  137 +++++++++++++++++++++++++++++----------------------
 fs/nfsd/nfsd.h      |    4 +
 fs/nfsd/nfssvc.c    |   15 +++++-
 4 files changed, 99 insertions(+), 60 deletions(-)
#1 lore raw

PATCH 1/5 nfsd: make client_lock per net

This lock protects the client lru list and session hash table, which are allocated per network namespace already.

---
 fs/nfsd/netns.h     |    3 ++
 fs/nfsd/nfs4state.c |   73 ++++++++++++++++++++++++++++++---------------------
 2 files changed, 46 insertions(+), 30 deletions(-)
fs/nfsd/netns.h +3 -0
index 227b93e..08d5fa1 100644
@@ -81,6 +81,9 @@ struct nfsd_net {
 	struct list_head close_lru;
 
 	struct delayed_work laundromat_work;
+
+	/* client_lock protects the client lru list and session hash table */
+	spinlock_t client_lock;
 };
 
 extern int nfsd_net_id;
fs/nfsd/nfs4state.c +43 -30

237 lines not on this page · raw message

#2 lore raw

PATCH 2/5 nfsd: make delegations shutdown network namespace aware

In reply to Stanislav Kinsbursky

NFSv4 delegations are stored in global list. But they are nfs4_client dependent, which is network namespace aware already.
State shutdown and laundromat are done per network namespace as well. So, delegations unhash have to be done in network namespace context.

---
 fs/nfsd/nfs4state.c |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)
fs/nfsd/nfs4state.c +4 -0
index bf6edeb..149f7b1 100644
@@ -3230,6 +3230,8 @@ nfs4_laundromat(struct nfsd_net *nn)
 	spin_lock(&recall_lock);
 	list_for_each_safe(pos, next, &del_recall_lru) {
 		dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru);
+		if (net_generic(dp->dl_stid.sc_client->net, nfsd_net_id) != nn)
+			continue;
 		if (time_after((unsigned long)dp->dl_time, (unsigned long)cutoff)) {
 			u = dp->dl_time - cutoff;
 			if (test_val > u)
@@ -4918,6 +4920,8 @@ __nfs4_state_shutdown(struct net *net)
 	spin_lock(&recall_lock);
 	list_for_each_safe(pos, next, &del_recall_lru) {
 		dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru);
+		if (dp->dl_stid.sc_client->net != net)
+			continue;
 		list_move(&dp->dl_recall_lru, &reaplist);
 	}
 	spin_unlock(&recall_lock);
#3 lore raw

PATCH 3/5 nfsd: cleanup NFSd state shutdown a bit

In reply to Stanislav Kinsbursky

This patch renames __nfs4_state_shutdown_net() into nfs4_state_shutdown_net(), __nfs4_state_shutdown() into nfs4_state_shutdown_net() and moves all network related shutdown operations to nfs4_state_shutdown_net().

---
 fs/nfsd/nfs4state.c |   19 +++++++++----------
 1 files changed, 9 insertions(+), 10 deletions(-)
fs/nfsd/nfs4state.c +9 -10
index 149f7b1..edff2cc 100644
@@ -4825,7 +4825,7 @@ err:
 }
 
 static void
-__nfs4_state_shutdown_net(struct net *net)
+nfs4_state_destroy_net(struct net *net)
 {
 	int i;
 	struct nfs4_client *clp = NULL;
@@ -4853,6 +4853,7 @@ __nfs4_state_shutdown_net(struct net *net)
 	kfree(nn->ownerstr_hashtbl);
 	kfree(nn->unconf_id_hashtbl);
 	kfree(nn->conf_id_hashtbl);
+	put_net(net);
 }
 
 /* initialization to perform when the nfsd service is started: */
@@ -4902,19 +4903,20 @@ out_free_laundry:
 	destroy_workqueue(laundry_wq);
 out_recovery:
 	nfsd4_client_tracking_exit(net);
-	__nfs4_state_shutdown_net(net);
-	put_net(net);
+	nfs4_state_destroy_net(net);
 	return ret;
 }
 
 /* should be called with the state lock held */
 static void
-__nfs4_state_shutdown(struct net *net)
+nfs4_state_shutdown_net(struct net *net)
 {
 	struct nfs4_delegation *dp = NULL;
 	struct list_head *pos, *next, reaplist;
+	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
-	__nfs4_state_shutdown_net(net);
+	cancel_delayed_work_sync(&nn->laundromat_work);
+	locks_end_grace(&nn->nfsd4_manager);
 
 	INIT_LIST_HEAD(&reaplist);
 	spin_lock(&recall_lock);
@@ -4931,19 +4933,16 @@ __nfs4_state_shutdown(struct net *net)
 	}
 
 	nfsd4_client_tracking_exit(net);
-	put_net(net);
+	nfs4_state_destroy_net(net);
 }
 
 void
 nfs4_state_shutdown(void)
 {
 	struct net *net = &init_net;
-	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
-	cancel_delayed_work_sync(&nn->laundromat_work);
+	nfs4_state_shutdown_net(net);
 	destroy_workqueue(laundry_wq);
-	locks_end_grace(&nn->nfsd4_manager);
-	__nfs4_state_shutdown(net);
 	nfsd4_destroy_callback_queue();
 }
 
#4 lore raw

PATCH 4/5 nfsd: cleanup NFSd state start a bit

In reply to Stanislav Kinsbursky

This patch renames nfs4_state_start_net() into nfs4_state_create_net(), where get_net() now performed.
Also it introduces new nfs4_state_start_net(), which is now responsible for state creation and initializing all per-net data and which is now called from nfs4_state_start().

---
 fs/nfsd/nfs4state.c |   59 ++++++++++++++++++++++++++++++---------------------
 1 files changed, 35 insertions(+), 24 deletions(-)
fs/nfsd/nfs4state.c +35 -24
index edff2cc..72df721 100644
@@ -4766,7 +4766,7 @@ set_max_delegations(void)
 	max_delegations = nr_free_buffer_pages() >> (20 - 2 - PAGE_SHIFT);
 }
 
-static int nfs4_state_start_net(struct net *net)
+static int nfs4_state_create_net(struct net *net)
 {
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 	int i;
@@ -4809,6 +4809,7 @@ static int nfs4_state_start_net(struct net *net)
 	spin_lock_init(&nn->client_lock);
 
 	INIT_DELAYED_WORK(&nn->laundromat_work, laundromat_main);
+	get_net(net);
 
 	return 0;
 
@@ -4856,37 +4857,35 @@ nfs4_state_destroy_net(struct net *net)
 	put_net(net);
 }
 
-/* initialization to perform when the nfsd service is started: */
-
-int
-nfs4_state_start(void)
+static int
+nfs4_state_start_net(struct net *net)
 {
-	struct net *net = &init_net;
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 	int ret;
 
-	/*
-	 * FIXME: For now, we hang most of the pernet global stuff off of
-	 * init_net until nfsd is fully containerized. Eventually, we'll
-	 * need to pass a net pointer into this function, take a reference
-	 * to that instead and then do most of the rest of this on a per-net
-	 * basis.
-	 */
-	get_net(net);
-	ret = nfs4_state_start_net(net);
+	ret = nfs4_state_create_net(net);
 	if (ret)
 		return ret;
 	nfsd4_client_tracking_init(net);
 	nn->boot_time = get_seconds();
 	locks_start_grace(net, &nn->nfsd4_manager);
 	nn->grace_ended = false;
-	printk(KERN_INFO "NFSD: starting %ld-second grace period\n",
-	       nfsd4_grace);
+	printk(KERN_INFO "NFSD: starting %ld-second grace period (net %p)\n",
+	       nfsd4_grace, net);
+	queue_delayed_work(laundry_wq, &nn->laundromat_work, nfsd4_grace * HZ);
+	return 0;
+}
+
+/* initialization to perform when the nfsd service is started: */
+
+int
+nfs4_state_start(void)
+{
+	int ret;
+
 	ret = set_callback_cred();
-	if (ret) {
-		ret = -ENOMEM;
-		goto out_recovery;
-	}
+	if (ret)
+		return -ENOMEM;
 	laundry_wq = create_singlethread_workqueue("nfsd4");
 	if (laundry_wq == NULL) {
 		ret = -ENOMEM;
@@ -4896,14 +4895,26 @@ nfs4_state_start(void)
 	if (ret)
 		goto out_free_laundry;
 
-	queue_delayed_work(laundry_wq, &nn->laundromat_work, nfsd4_grace * HZ);
 	set_max_delegations();
+
+	/*
+	 * FIXME: For now, we hang most of the pernet global stuff off of
+	 * init_net until nfsd is fully containerized. Eventually, we'll
+	 * need to pass a net pointer into this function, take a reference
+	 * to that instead and then do most of the rest of this on a per-net
+	 * basis.
+	 */
+	ret = nfs4_state_start_net(&init_net);
+	if (ret)
+		goto out_free_callback;
+
 	return 0;
+
+out_free_callback:
+	nfsd4_destroy_callback_queue();
 out_free_laundry:
 	destroy_workqueue(laundry_wq);
 out_recovery:
-	nfsd4_client_tracking_exit(net);
-	nfs4_state_destroy_net(net);
 	return ret;
 }
 
#5 lore raw

PATCH 5/5 nfsd: call state init and shutdown twice

In reply to Stanislav Kinsbursky

Split NFSv4 state init and shutdown into two different calls: per-net one and generic one.
Per-net cwinit/shutdown pair have to be called for any namespace, generic pair
- only once on NSFd kthreads start and shutdown respectively.

Refresh of diff-nfsd-call-state-init-twice

---
 fs/nfsd/nfs4state.c |   30 ++++++++++++------------------
 fs/nfsd/nfsd.h      |    4 ++++
 fs/nfsd/nfssvc.c    |   15 +++++++++++++--
 3 files changed, 29 insertions(+), 20 deletions(-)
fs/nfsd/nfs4state.c +12 -18
index 72df721..93c7882 100644
@@ -4857,12 +4857,22 @@ nfs4_state_destroy_net(struct net *net)
 	put_net(net);
 }
 
-static int
+int
 nfs4_state_start_net(struct net *net)
 {
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 	int ret;
 
+	/*
+	 * FIXME: For now, we hang most of the pernet global stuff off of
+	 * init_net until nfsd is fully containerized. Eventually, we'll
+	 * need to pass a net pointer into this function, take a reference
+	 * to that instead and then do most of the rest of this on a per-net
+	 * basis.
+	 */
+	if (net != &init_net)
+		return -EINVAL;
+
 	ret = nfs4_state_create_net(net);
 	if (ret)
 		return ret;
@@ -4897,21 +4907,8 @@ nfs4_state_start(void)
 
 	set_max_delegations();
 
-	/*
-	 * FIXME: For now, we hang most of the pernet global stuff off of
-	 * init_net until nfsd is fully containerized. Eventually, we'll
-	 * need to pass a net pointer into this function, take a reference
-	 * to that instead and then do most of the rest of this on a per-net
-	 * basis.
-	 */
-	ret = nfs4_state_start_net(&init_net);
-	if (ret)
-		goto out_free_callback;
-
 	return 0;
 
-out_free_callback:
-	nfsd4_destroy_callback_queue();
 out_free_laundry:
 	destroy_workqueue(laundry_wq);
 out_recovery:
@@ -4919,7 +4916,7 @@ out_recovery:
 }
 
 /* should be called with the state lock held */
-static void
+void
 nfs4_state_shutdown_net(struct net *net)
 {
 	struct nfs4_delegation *dp = NULL;
@@ -4950,9 +4947,6 @@ nfs4_state_shutdown_net(struct net *net)
 void
 nfs4_state_shutdown(void)
 {
-	struct net *net = &init_net;
-
-	nfs4_state_shutdown_net(net);
 	destroy_workqueue(laundry_wq);
 	nfsd4_destroy_callback_queue();
 }
fs/nfsd/nfsd.h +4 -0
index 80d5ce4..d7b210b 100644
@@ -121,7 +121,9 @@ void nfs4_state_init(void);
 int nfsd4_init_slabs(void);
 void nfsd4_free_slabs(void);
 int nfs4_state_start(void);
+int nfs4_state_start_net(struct net *net);
 void nfs4_state_shutdown(void);
+void nfs4_state_shutdown_net(struct net *net);
 void nfs4_reset_lease(time_t leasetime);
 int nfs4_reset_recoverydir(char *recdir);
 char * nfs4_recoverydir(void);
@@ -130,7 +132,9 @@ static inline void nfs4_state_init(void) { }
 static inline int nfsd4_init_slabs(void) { return 0; }
 static inline void nfsd4_free_slabs(void) { }
 static inline int nfs4_state_start(void) { return 0; }
+static inline int nfs4_state_start_net(struct net *net) { return 0; }
 static inline void nfs4_state_shutdown(void) { }
+static inline void nfs4_state_shutdown_net(struct net *net) { }
 static inline void nfs4_reset_lease(time_t leasetime) { }
 static inline int nfs4_reset_recoverydir(char *recdir) { return 0; }
 static inline char * nfs4_recoverydir(void) {return NULL; }
fs/nfsd/nfssvc.c +13 -2
index 2013aa0..6dffa74 100644
@@ -207,6 +207,7 @@ static bool nfsd_up = false;
 static int nfsd_startup(int nrservs)
 {
 	int ret;
+	struct net *net = &init_net;
 
 	if (nfsd_up)
 		return 0;
@@ -221,14 +222,21 @@ static int nfsd_startup(int nrservs)
 	ret = nfsd_init_socks();
 	if (ret)
 		goto out_racache;
-	ret = lockd_up(&init_net);
+	ret = lockd_up(net);
 	if (ret)
 		goto out_racache;
 	ret = nfs4_state_start();
 	if (ret)
 		goto out_lockd;
+
+	ret = nfs4_state_start_net(net);
+	if (ret)
+		goto out_net_state;
+
 	nfsd_up = true;
 	return 0;
+out_net_state:
+	nfs4_state_shutdown();
 out_lockd:
 	lockd_down(&init_net);
 out_racache:
@@ -238,6 +246,8 @@ out_racache:
 
 static void nfsd_shutdown(void)
 {
+	struct net *net = &init_net;
+
 	/*
 	 * write_ports can create the server without actually starting
 	 * any threads--if we get shut down before any threads are
@@ -246,8 +256,9 @@ static void nfsd_shutdown(void)
 	 */
 	if (!nfsd_up)
 		return;
+	nfs4_state_shutdown_net(net);
 	nfs4_state_shutdown();
-	lockd_down(&init_net);
+	lockd_down(net);
 	nfsd_racache_shutdown();
 	nfsd_up = false;
 }
#6 lore raw
In reply to Stanislav Kinsbursky
Quoted 6 lines · Stanislav Kinsbursky

This patch set makes NFSv4 state created and destroyed per net and thus completes it's containerization (the only exceprtion is client_mutex, but this one a hard nut to crack).

Note: patch set depend on previously sent patch named "nfsd4: remove state lock from nfs4_state_shutdown".

Looks OK.--b.

Quoted 19 lines

The following series implements...

---

Stanislav Kinsbursky (5):
nfsd: make client_lock per net
nfsd: make delegations shutdown network namespace aware
nfsd: cleanup NFSd state shutdown a bit
nfsd: cleanup NFSd state start a bit
nfsd: call state init and shutdown twice

 fs/nfsd/netns.h     |    3 +
 fs/nfsd/nfs4state.c |  137 +++++++++++++++++++++++++++++----------------------
 fs/nfsd/nfsd.h      |    4 +
 fs/nfsd/nfssvc.c    |   15 +++++-
 4 files changed, 99 insertions(+), 60 deletions(-)