nfsd: more NFSv4 state containerization
Subsystems (2)
cover nfsd: more NFSv4 state containerization
This patch set makes NFSv4 state created and destroyed per net and thus completes it's containerization (the only exceprtion is client_mutex, but this one a hard nut to crack).
Note: patch set depend on previously sent patch named "nfsd4: remove state lock from nfs4_state_shutdown".
The following series implements...
---
Stanislav Kinsbursky (5):
nfsd: make client_lock per net
nfsd: make delegations shutdown network namespace aware
nfsd: cleanup NFSd state shutdown a bit
nfsd: cleanup NFSd state start a bit
nfsd: call state init and shutdown twice
fs/nfsd/netns.h | 3 + fs/nfsd/nfs4state.c | 137 +++++++++++++++++++++++++++++---------------------- fs/nfsd/nfsd.h | 4 + fs/nfsd/nfssvc.c | 15 +++++- 4 files changed, 99 insertions(+), 60 deletions(-)
PATCH 1/5 nfsd: make client_lock per net
This lock protects the client lru list and session hash table, which are allocated per network namespace already.
--- fs/nfsd/netns.h | 3 ++ fs/nfsd/nfs4state.c | 73 ++++++++++++++++++++++++++++++--------------------- 2 files changed, 46 insertions(+), 30 deletions(-)
fs/nfsd/netns.h +3 -0
@@ -81,6 +81,9 @@ struct nfsd_net {struct list_head close_lru;struct delayed_work laundromat_work; + +/* client_lock protects the client lru list and session hash table */ +spinlock_t client_lock; };extern int nfsd_net_id;
fs/nfsd/nfs4state.c +43 -30
237 lines not on this page · raw message
PATCH 2/5 nfsd: make delegations shutdown network namespace aware
In reply to Stanislav KinsburskyNFSv4 delegations are stored in global list. But they are nfs4_client dependent, which is network namespace aware already.
State shutdown and laundromat are done per network namespace as well. So, delegations unhash have to be done in network namespace context.
--- fs/nfsd/nfs4state.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-)
fs/nfsd/nfs4state.c +4 -0
@@ -3230,6 +3230,8 @@ nfs4_laundromat(struct nfsd_net *nn)spin_lock (&recall_lock);list_for_each_safe (pos, next, &del_recall_lru) { dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru); +if (net_generic (dp->dl_stid.sc_client->net, nfsd_net_id) != nn) +continue ;if (time_after ((unsigned long )dp->dl_time, (unsigned long )cutoff)) { u = dp->dl_time - cutoff;if (test_val > u)
@@ -4918,6 +4920,8 @@ __nfs4_state_shutdown(struct net *net)spin_lock (&recall_lock);list_for_each_safe (pos, next, &del_recall_lru) { dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru); +if (dp->dl_stid.sc_client->net != net) +continue ;list_move (&dp->dl_recall_lru, &reaplist); }spin_unlock (&recall_lock);
PATCH 3/5 nfsd: cleanup NFSd state shutdown a bit
In reply to Stanislav KinsburskyThis patch renames __nfs4_state_shutdown_net() into nfs4_state_shutdown_net(), __nfs4_state_shutdown() into nfs4_state_shutdown_net() and moves all network related shutdown operations to nfs4_state_shutdown_net().
--- fs/nfsd/nfs4state.c | 19 +++++++++---------- 1 files changed, 9 insertions(+), 10 deletions(-)
fs/nfsd/nfs4state.c +9 -10
@@ -4825,7 +4825,7 @@ err: }static void -__nfs4_state_shutdown_net (struct net *net) +nfs4_state_destroy_net (struct net *net) {int i;struct nfs4_client *clp =NULL ;
@@ -4853,6 +4853,7 @@ __nfs4_state_shutdown_net(struct net *net)kfree (nn->ownerstr_hashtbl);kfree (nn->unconf_id_hashtbl);kfree (nn->conf_id_hashtbl); +put_net (net); }/* initialization to perform when the nfsd service is started: */
@@ -4902,19 +4903,20 @@ out_free_laundry:destroy_workqueue (laundry_wq); out_recovery:nfsd4_client_tracking_exit (net); -__nfs4_state_shutdown_net (net); -put_net (net); +nfs4_state_destroy_net (net);return ret; }/* should be called with the state lock held */ static void -__nfs4_state_shutdown (struct net *net) +nfs4_state_shutdown_net (struct net *net) {struct nfs4_delegation *dp =NULL ;struct list_head *pos, *next, reaplist; +struct nfsd_net *nn =net_generic (net, nfsd_net_id); -__nfs4_state_shutdown_net (net); +cancel_delayed_work_sync (&nn->laundromat_work); +locks_end_grace (&nn->nfsd4_manager);INIT_LIST_HEAD (&reaplist);spin_lock (&recall_lock);
@@ -4931,19 +4933,16 @@ __nfs4_state_shutdown(struct net *net) }nfsd4_client_tracking_exit (net); -put_net (net); +nfs4_state_destroy_net (net); }void nfs4_state_shutdown (void ) {struct net *net = &init_net; -struct nfsd_net *nn =net_generic (net, nfsd_net_id); -cancel_delayed_work_sync (&nn->laundromat_work); +nfs4_state_shutdown_net (net);destroy_workqueue (laundry_wq); -locks_end_grace (&nn->nfsd4_manager); -__nfs4_state_shutdown (net);nfsd4_destroy_callback_queue (); }
PATCH 4/5 nfsd: cleanup NFSd state start a bit
In reply to Stanislav KinsburskyThis patch renames nfs4_state_start_net() into nfs4_state_create_net(), where get_net() now performed.
Also it introduces new nfs4_state_start_net(), which is now responsible for state creation and initializing all per-net data and which is now called from nfs4_state_start().
--- fs/nfsd/nfs4state.c | 59 ++++++++++++++++++++++++++++++--------------------- 1 files changed, 35 insertions(+), 24 deletions(-)
fs/nfsd/nfs4state.c +35 -24
@@ -4766,7 +4766,7 @@ set_max_delegations(void) max_delegations =nr_free_buffer_pages () >> (20 -2 -PAGE_SHIFT ); } -static int nfs4_state_start_net (struct net *net) +static int nfs4_state_create_net (struct net *net) {struct nfsd_net *nn =net_generic (net, nfsd_net_id);int i;
@@ -4809,6 +4809,7 @@ static int nfs4_state_start_net(struct net *net)spin_lock_init (&nn->client_lock);INIT_DELAYED_WORK (&nn->laundromat_work, laundromat_main); +get_net (net);return 0 ;
@@ -4856,37 +4857,35 @@ nfs4_state_destroy_net(struct net *net)put_net (net); } -/* initialization to perform when the nfsd service is started: */ - -int -nfs4_state_start (void ) +static int +nfs4_state_start_net (struct net *net) { -struct net *net = &init_net;struct nfsd_net *nn =net_generic (net, nfsd_net_id);int ret; -/* -* FIXME: For now, we hang most of the pernet global stuff off of -* init_net until nfsd is fully containerized. Eventually, we'll -* need to pass a net pointer into this function, take a reference -* to that instead and then do most of the rest of this on a per-net -* basis. -*/ -get_net (net); - ret =nfs4_state_start_net (net); + ret =nfs4_state_create_net (net);if (ret)return ret;nfsd4_client_tracking_init (net); nn->boot_time =get_seconds ();locks_start_grace (net, &nn->nfsd4_manager); nn->grace_ended = false; -printk (KERN_INFO "NFSD: starting %ld-second grace period\n" , - nfsd4_grace); +printk (KERN_INFO "NFSD: starting %ld-second grace period (net %p)\n" , + nfsd4_grace, net); +queue_delayed_work (laundry_wq, &nn->laundromat_work, nfsd4_grace *HZ ); +return 0 ; +} + +/* initialization to perform when the nfsd service is started: */ + +int +nfs4_state_start (void ) +{ +int ret; + ret =set_callback_cred (); -if (ret) { - ret = -ENOMEM ; - goto out_recovery; - } +if (ret) +return -ENOMEM ; laundry_wq =create_singlethread_workqueue ("nfsd4" );if (laundry_wq ==NULL ) { ret = -ENOMEM ;
@@ -4896,14 +4895,26 @@ nfs4_state_start(void)if (ret) goto out_free_laundry; -queue_delayed_work (laundry_wq, &nn->laundromat_work, nfsd4_grace *HZ );set_max_delegations (); + +/* +* FIXME: For now, we hang most of the pernet global stuff off of +* init_net until nfsd is fully containerized. Eventually, we'll +* need to pass a net pointer into this function, take a reference +* to that instead and then do most of the rest of this on a per-net +* basis. +*/ + ret =nfs4_state_start_net (&init_net); +if (ret) + goto out_free_callback; +return 0 ; + +out_free_callback: +nfsd4_destroy_callback_queue (); out_free_laundry:destroy_workqueue (laundry_wq); out_recovery: -nfsd4_client_tracking_exit (net); -nfs4_state_destroy_net (net);return ret; }
PATCH 5/5 nfsd: call state init and shutdown twice
In reply to Stanislav KinsburskySplit NFSv4 state init and shutdown into two different calls: per-net one and generic one.
Per-net cwinit/shutdown pair have to be called for any namespace, generic pair
- only once on NSFd kthreads start and shutdown respectively.
Refresh of diff-nfsd-call-state-init-twice
--- fs/nfsd/nfs4state.c | 30 ++++++++++++------------------ fs/nfsd/nfsd.h | 4 ++++ fs/nfsd/nfssvc.c | 15 +++++++++++++-- 3 files changed, 29 insertions(+), 20 deletions(-)
fs/nfsd/nfs4state.c +12 -18
@@ -4857,12 +4857,22 @@ nfs4_state_destroy_net(struct net *net)put_net (net); } -static int +int nfs4_state_start_net (struct net *net) {struct nfsd_net *nn =net_generic (net, nfsd_net_id);int ret; +/* +* FIXME: For now, we hang most of the pernet global stuff off of +* init_net until nfsd is fully containerized. Eventually, we'll +* need to pass a net pointer into this function, take a reference +* to that instead and then do most of the rest of this on a per-net +* basis. +*/ +if (net != &init_net) +return -EINVAL ; + ret =nfs4_state_create_net (net);if (ret)return ret;
@@ -4897,21 +4907,8 @@ nfs4_state_start(void)set_max_delegations (); -/* -* FIXME: For now, we hang most of the pernet global stuff off of -* init_net until nfsd is fully containerized. Eventually, we'll -* need to pass a net pointer into this function, take a reference -* to that instead and then do most of the rest of this on a per-net -* basis. -*/ - ret =nfs4_state_start_net (&init_net); -if (ret) - goto out_free_callback; -return 0 ; -out_free_callback: -nfsd4_destroy_callback_queue (); out_free_laundry:destroy_workqueue (laundry_wq); out_recovery:
@@ -4919,7 +4916,7 @@ out_recovery: }/* should be called with the state lock held */ -static void +void nfs4_state_shutdown_net (struct net *net) {struct nfs4_delegation *dp =NULL ;
@@ -4950,9 +4947,6 @@ nfs4_state_shutdown_net(struct net *net)void nfs4_state_shutdown (void ) { -struct net *net = &init_net; - -nfs4_state_shutdown_net (net);destroy_workqueue (laundry_wq);nfsd4_destroy_callback_queue (); }
fs/nfsd/nfsd.h +4 -0
@@ -121,7 +121,9 @@ void nfs4_state_init(void);int nfsd4_init_slabs (void );void nfsd4_free_slabs (void );int nfs4_state_start (void ); +int nfs4_state_start_net (struct net *net);void nfs4_state_shutdown (void ); +void nfs4_state_shutdown_net (struct net *net);void nfs4_reset_lease (time_t leasetime);int nfs4_reset_recoverydir (char *recdir);char *nfs4_recoverydir (void );
@@ -130,7 +132,9 @@ static inline void nfs4_state_init(void) { }static inline int nfsd4_init_slabs (void ) {return 0 ; }static inline void nfsd4_free_slabs (void ) { }static inline int nfs4_state_start (void ) {return 0 ; } +static inline int nfs4_state_start_net (struct net *net) {return 0 ; }static inline void nfs4_state_shutdown (void ) { } +static inline void nfs4_state_shutdown_net (struct net *net) { }static inline void nfs4_reset_lease (time_t leasetime) { }static inline int nfs4_reset_recoverydir (char *recdir) {return 0 ; }static inline char *nfs4_recoverydir (void ) {return NULL ; }
fs/nfsd/nfssvc.c +13 -2
@@ -207,6 +207,7 @@ static bool nfsd_up = false;static int nfsd_startup (int nrservs) {int ret; +struct net *net = &init_net;if (nfsd_up)return 0 ;
@@ -221,14 +222,21 @@ static int nfsd_startup(int nrservs) ret =nfsd_init_socks ();if (ret) goto out_racache; - ret =lockd_up (&init_net); + ret =lockd_up (net);if (ret) goto out_racache; ret =nfs4_state_start ();if (ret) goto out_lockd; + + ret =nfs4_state_start_net (net); +if (ret) + goto out_net_state; + nfsd_up = true;return 0 ; +out_net_state: +nfs4_state_shutdown (); out_lockd:lockd_down (&init_net); out_racache:
@@ -238,6 +246,8 @@ out_racache:static void nfsd_shutdown (void ) { +struct net *net = &init_net; +/* * write_ports can create the server without actually starting * any threads--if we get shut down before any threads are
@@ -246,8 +256,9 @@ static void nfsd_shutdown(void)*/ if (!nfsd_up)return ; +nfs4_state_shutdown_net (net);nfs4_state_shutdown (); -lockd_down (&init_net); +lockd_down (net);nfsd_racache_shutdown (); nfsd_up = false; }
Quoted 6 lines · Stanislav Kinsbursky
This patch set makes NFSv4 state created and destroyed per net and thus completes it's containerization (the only exceprtion is
client_mutex, but this one a hard nut to crack).Note: patch set depend on previously sent patch named "nfsd4: remove state lock from
nfs4_state_shutdown".
Looks OK.--b.
Quoted 19 lines
The following series implements...
---
Stanislav Kinsbursky (5):
nfsd: makeclient_lockper net
nfsd: make delegations shutdown network namespace aware
nfsd: cleanup NFSd state shutdown a bit
nfsd: cleanup NFSd state start a bit
nfsd: call state init and shutdown twicefs/nfsd/netns.h | 3 + fs/nfsd/nfs4state.c | 137 +++++++++++++++++++++++++++++---------------------- fs/nfsd/nfsd.h | 4 + fs/nfsd/nfssvc.c | 15 +++++- 4 files changed, 99 insertions(+), 60 deletions(-)