use a hash for looking up delegation v3
Subsystems (2)
Hi all,
currently recalling delegations has to walk the server->delegations list, and then take the lock for each delegation. This can take a lot of time and has adverse effects to the rest of the system due to the number atomic operations, cache lines touched and a long RCU critical section.
This series first converts the delegation watermark to be per-server, as all the state guarded by it is per-server and the commit message adding it talks about server side overhead as well, and then adds a very simple hash for finding the delegation for a given file handle in nfs_delegation_find_inode_server.
With this hash sample microbenchmarks that cause delegation recalls in reverse list order are sped up ~5 percent, although the time is still very variable due to other factors.
Changes since v2:
- keep the hash allocation for cloning inside the nfsv4 module
Changes since v1:
- only allocate the delegation hash for v4 mounts
Diffstat:
fs/nfs/client.c | 1 fs/nfs/delegation.c | 110 ++++++++++++++++++++++++++++------------------ fs/nfs/delegation.h | 3 + fs/nfs/nfs4client.c | 14 +++-- fs/nfs/nfs4proc.c | 22 ++++++++- include/linux/nfs_fs_sb.h | 3 + 6 files changed, 106 insertions(+), 47 deletions(-)
PATCH 1/5 NFS: cleanup error handling in nfs4_server_common_setup
Return error directly instead of using a goto label for it.
--- fs/nfs/nfs4client.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-)
fs/nfs/nfs4client.c +4 -5
@@ -1103,14 +1103,14 @@ static int nfs4_server_common_setup(struct nfs_server *server,/* We must ensure the session is initialised first */ error =nfs4_init_session (server->nfs_client);if (error <0 ) - goto out; +return error;nfs4_server_set_init_caps (server);/* Probe the root fh to retrieve its FSID and filehandle */ error =nfs4_get_rootfh (server, mntfh, auth_probe);if (error <0 ) - goto out; +return error; dprintk("Server FSID: %llx:%llx\n", (unsigned long long) server->fsid.major,
@@ -1119,7 +1119,7 @@ static int nfs4_server_common_setup(struct nfs_server *server, error =nfs_probe_server (server, mntfh);if (error <0 ) - goto out; +return error;nfs4_session_limit_rwsize (server);nfs4_session_limit_xasize (server);
@@ -1130,8 +1130,7 @@ static int nfs4_server_common_setup(struct nfs_server *server,nfs_server_insert_lists (server); server->mount_time = jiffies; server->destroy = nfs4_destroy_server; -out: -return error; +return 0 ; }/*
Signature · 1 line
2.47.2
PATCH 2/5 NFS: cleanup nfs_inode_reclaim_delegation
In reply to Christoph HellwigReduce a level of indentation for most of the code in this function.
--- fs/nfs/delegation.c | 48 ++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 24 deletions(-)
fs/nfs/delegation.c +24 -24
@@ -237,34 +237,34 @@ void nfs_inode_reclaim_delegation(struct inode *inode, const struct cred *cred,rcu_read_lock (); delegation =rcu_dereference (NFS_I (inode)->delegation); -if (delegation !=NULL ) { -spin_lock (&delegation->lock); -nfs4_stateid_copy (&delegation->stateid, stateid); - delegation->type = type; - delegation->pagemod_limit = pagemod_limit; - oldcred = delegation->cred; - delegation->cred =get_cred (cred); -switch (deleg_type) { -case NFS4_OPEN_DELEGATE_READ_ATTRS_DELEG : -case NFS4_OPEN_DELEGATE_WRITE_ATTRS_DELEG : -set_bit (NFS_DELEGATION_DELEGTIME , &delegation->flags); -break ; -default : -clear_bit (NFS_DELEGATION_DELEGTIME , &delegation->flags); - } -clear_bit (NFS_DELEGATION_NEED_RECLAIM , &delegation->flags); -if (test_and_clear_bit (NFS_DELEGATION_REVOKED , - &delegation->flags)) -atomic_long_inc (&nfs_active_delegations); -spin_unlock (&delegation->lock); -rcu_read_unlock (); -put_cred (oldcred); -trace_nfs4_reclaim_delegation (inode, type); - }else { +if (!delegation) {rcu_read_unlock ();nfs_inode_set_delegation (inode, cred, type, stateid, pagemod_limit, deleg_type); +return ; + } + +spin_lock (&delegation->lock); +nfs4_stateid_copy (&delegation->stateid, stateid); + delegation->type = type; + delegation->pagemod_limit = pagemod_limit; + oldcred = delegation->cred; + delegation->cred =get_cred (cred); +switch (deleg_type) { +case NFS4_OPEN_DELEGATE_READ_ATTRS_DELEG : +case NFS4_OPEN_DELEGATE_WRITE_ATTRS_DELEG : +set_bit (NFS_DELEGATION_DELEGTIME , &delegation->flags); +break ; +default : +clear_bit (NFS_DELEGATION_DELEGTIME , &delegation->flags); } +clear_bit (NFS_DELEGATION_NEED_RECLAIM , &delegation->flags); +if (test_and_clear_bit (NFS_DELEGATION_REVOKED , &delegation->flags)) +atomic_long_inc (&nfs_active_delegations); +spin_unlock (&delegation->lock); +rcu_read_unlock (); +put_cred (oldcred); +trace_nfs4_reclaim_delegation (inode, type); } static int nfs_do_return_delegation(struct inode *inode,
Signature · 1 line
2.47.2
PATCH 3/5 NFS: move the delegation_watermark module parameter
In reply to Christoph HellwigKeep the module_param_named next to the variable declaration instead of somewhere unrelated, following the best practice in the rest of the kernel.
--- fs/nfs/delegation.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
fs/nfs/delegation.c +1 -2
@@ -29,6 +29,7 @@static atomic_long_t nfs_active_delegations;static unsigned nfs_delegation_watermark =NFS_DEFAULT_DELEGATION_WATERMARK ; +module_param_named (delegation_watermark, nfs_delegation_watermark, uint,0644 );static void __nfs_free_delegation (struct nfs_delegation *delegation) {
@@ -1573,5 +1574,3 @@ bool nfs4_delegation_flush_on_close(const struct inode *inode)rcu_read_unlock ();return ret; } - -module_param_named (delegation_watermark, nfs_delegation_watermark, uint,0644 );
Signature · 1 line
2.47.2
PATCH 4/5 NFS: track active delegations per-server
In reply to Christoph HellwigThe active delegation watermark was added to avoid overloading servers. Track the active delegation per-server instead of globally so that clients talking to multiple servers aren't limited by the global limit.
--- fs/nfs/client.c | 1 + fs/nfs/delegation.c | 35 +++++++++++++++++++---------------- include/linux/nfs_fs_sb.h | 1 + 3 files changed, 21 insertions(+), 16 deletions(-)
fs/nfs/client.c +1 -0
@@ -1005,6 +1005,7 @@ struct nfs_server *nfs_alloc_server(void)INIT_LIST_HEAD (&server->ss_src_copies);atomic_set (&server->active,0 ); +atomic_long_set (&server->nr_active_delegations,0 ); server->io_stats =nfs_alloc_iostats ();if (!server->io_stats) {
fs/nfs/delegation.c +19 -16
@@ -27,7 +27,6 @@#define NFS_DEFAULT_DELEGATION_WATERMARK (5000U) -static atomic_long_t nfs_active_delegations;static unsigned nfs_delegation_watermark =NFS_DEFAULT_DELEGATION_WATERMARK ;module_param_named (delegation_watermark, nfs_delegation_watermark, uint,0644 );
@@ -38,11 +37,12 @@ static void __nfs_free_delegation(struct nfs_delegation *delegation)kfree_rcu (delegation, rcu); } -static void nfs_mark_delegation_revoked (struct nfs_delegation *delegation) +static void nfs_mark_delegation_revoked (struct nfs_server *server, +struct nfs_delegation *delegation) {if (!test_and_set_bit (NFS_DELEGATION_REVOKED , &delegation->flags)) { delegation->stateid.type =NFS4_INVALID_STATEID_TYPE ; -atomic_long_dec (&nfs_active_delegations); +atomic_long_dec (&server->nr_active_delegations);if (!test_bit (NFS_DELEGATION_RETURNING , &delegation->flags))nfs_clear_verifier_delegated (delegation->inode); }
@@ -60,9 +60,10 @@ static void nfs_put_delegation(struct nfs_delegation *delegation)__nfs_free_delegation (delegation); } -static void nfs_free_delegation (struct nfs_delegation *delegation) +static void nfs_free_delegation (struct nfs_server *server, +struct nfs_delegation *delegation) { -nfs_mark_delegation_revoked (delegation); +nfs_mark_delegation_revoked (server, delegation);nfs_put_delegation (delegation); }
@@ -261,7 +262,7 @@ void nfs_inode_reclaim_delegation(struct inode *inode, const struct cred *cred, }clear_bit (NFS_DELEGATION_NEED_RECLAIM , &delegation->flags);if (test_and_clear_bit (NFS_DELEGATION_REVOKED , &delegation->flags)) -atomic_long_inc (&nfs_active_delegations); +atomic_long_inc (&NFS_SERVER (inode)->nr_active_delegations);spin_unlock (&delegation->lock);rcu_read_unlock ();put_cred (oldcred);
@@ -413,7 +414,8 @@ nfs_update_delegation_cred(struct nfs_delegation *delegation, }static void -nfs_update_inplace_delegation (struct nfs_delegation *delegation, +nfs_update_inplace_delegation (struct nfs_server *server, +struct nfs_delegation *delegation,const struct nfs_delegation *update) {if (nfs4_stateid_is_newer (&update->stateid, &delegation->stateid)) {
@@ -426,7 +428,7 @@ nfs_update_inplace_delegation(struct nfs_delegation *delegation,nfs_update_delegation_cred (delegation, update->cred);/* smp_mb__before_atomic() is implicit due to xchg() */ clear_bit (NFS_DELEGATION_REVOKED , &delegation->flags); -atomic_long_inc (&nfs_active_delegations); +atomic_long_inc (&server->nr_active_delegations); } } }
@@ -481,7 +483,7 @@ int nfs_inode_set_delegation(struct inode *inode, const struct cred *cred,if (nfs4_stateid_match_other (&old_delegation->stateid, &delegation->stateid)) {spin_lock (&old_delegation->lock); -nfs_update_inplace_delegation (old_delegation, +nfs_update_inplace_delegation (server, old_delegation, delegation);spin_unlock (&old_delegation->lock); goto out;
@@ -530,7 +532,7 @@ int nfs_inode_set_delegation(struct inode *inode, const struct cred *cred,rcu_assign_pointer (nfsi->delegation, delegation); delegation =NULL ; -atomic_long_inc (&nfs_active_delegations); +atomic_long_inc (&server->nr_active_delegations);trace_nfs4_set_delegation (inode, type);
@@ -544,7 +546,7 @@ int nfs_inode_set_delegation(struct inode *inode, const struct cred *cred,__nfs_free_delegation (delegation);if (freeme !=NULL ) {nfs_do_return_delegation (inode, freeme,0 ); -nfs_free_delegation (freeme); +nfs_free_delegation (server, freeme); }return status; }
@@ -756,7 +758,7 @@ void nfs_inode_evict_delegation(struct inode *inode)set_bit (NFS_DELEGATION_RETURNING , &delegation->flags);set_bit (NFS_DELEGATION_INODE_FREEING , &delegation->flags);nfs_do_return_delegation (inode, delegation,1 ); -nfs_free_delegation (delegation); +nfs_free_delegation (NFS_SERVER (inode), delegation); } }
@@ -842,7 +844,8 @@ void nfs4_inode_return_delegation_on_close(struct inode *inode)if (!delegation) goto out;if (test_bit (NFS_DELEGATION_RETURN_IF_CLOSED , &delegation->flags) || -atomic_long_read (&nfs_active_delegations) >= nfs_delegation_watermark) { +atomic_long_read (&NFS_SERVER (inode)->nr_active_delegations) >= + nfs_delegation_watermark) {spin_lock (&delegation->lock); if (delegation->inode && list_empty(&NFS_I(inode)->open_files) &&
@@ -1018,7 +1021,7 @@ static void nfs_revoke_delegation(struct inode *inode, }spin_unlock (&delegation->lock); } -nfs_mark_delegation_revoked (delegation); +nfs_mark_delegation_revoked (NFS_SERVER (inode), delegation); ret = true; out:rcu_read_unlock ();
@@ -1050,7 +1053,7 @@ void nfs_delegation_mark_returned(struct inode *inode, delegation->stateid.seqid = stateid->seqid; } -nfs_mark_delegation_revoked (delegation); +nfs_mark_delegation_revoked (NFS_SERVER (inode), delegation);clear_bit (NFS_DELEGATION_RETURNING , &delegation->flags);spin_unlock (&delegation->lock);if (nfs_detach_delegation (NFS_I (inode), delegation,NFS_SERVER (inode)))
@@ -1270,7 +1273,7 @@ static int nfs_server_reap_unclaimed_delegations(struct nfs_server *server,if (delegation !=NULL ) {if (nfs_detach_delegation (NFS_I (inode), delegation, server) !=NULL ) -nfs_free_delegation (delegation); +nfs_free_delegation (server, delegation);/* Match nfs_start_delegation_return_locked */ nfs_put_delegation (delegation); }
include/linux/nfs_fs_sb.h +1 -0
@@ -254,6 +254,7 @@ struct nfs_server {struct list_head state_owners_lru;struct list_head layouts;struct list_head delegations; +atomic_long_t nr_active_delegations;struct list_head ss_copies;struct list_head ss_src_copies;
Signature · 1 line
2.47.2
PATCH 5/5 NFS: use a hash table for delegation lookup
In reply to Christoph Hellwignfs_delegation_find_inode currently has to walk the entire list of delegations per inode, which can become pretty large, and can become even larger when increasing the delegation watermark.
Add a hash table to speed up the delegation lookup, sized as a fraction of the delegation watermark.
--- fs/nfs/delegation.c | 28 +++++++++++++++++++++++++++- fs/nfs/delegation.h | 3 +++ fs/nfs/nfs4client.c | 5 +++++ fs/nfs/nfs4proc.c | 22 +++++++++++++++++++++- include/linux/nfs_fs_sb.h | 2 ++ 5 files changed, 58 insertions(+), 2 deletions(-)
fs/nfs/delegation.c +27 -1
@@ -30,6 +30,13 @@static unsigned nfs_delegation_watermark =NFS_DEFAULT_DELEGATION_WATERMARK ;module_param_named (delegation_watermark, nfs_delegation_watermark, uint,0644 ); +static struct hlist_head *nfs_delegation_hash (struct nfs_server *server, +const struct nfs_fh *fhandle) +{ +return server->delegation_hash_table + + (nfs_fhandle_hash (fhandle) & server->delegation_hash_mask); +} +static void __nfs_free_delegation (struct nfs_delegation *delegation) {put_cred (delegation->cred);
@@ -367,6 +374,7 @@ nfs_detach_delegation_locked(struct nfs_inode *nfsi,spin_unlock (&delegation->lock);return NULL ; } +hlist_del_init_rcu (&delegation->hash);list_del_rcu (&delegation->super_list); delegation->inode =NULL ;rcu_assign_pointer (nfsi->delegation,NULL );
@@ -529,6 +537,8 @@ int nfs_inode_set_delegation(struct inode *inode, const struct cred *cred,spin_unlock (&inode->i_lock);list_add_tail_rcu (&delegation->super_list, &server->delegations); +hlist_add_head_rcu (&delegation->hash, +nfs_delegation_hash (server, &NFS_I (inode)->fh));rcu_assign_pointer (nfsi->delegation, delegation); delegation =NULL ;
@@ -1166,11 +1176,12 @@ static struct inode * nfs_delegation_find_inode_server(struct nfs_server *server, const struct nfs_fh *fhandle) { + struct hlist_head *head = nfs_delegation_hash(server, fhandle);struct nfs_delegation *delegation;struct super_block *freeme =NULL ;struct inode *res =NULL ; -list_for_each_entry_rcu (delegation, &server->delegations, super_list) { +hlist_for_each_entry_rcu (delegation, head, hash) {spin_lock (&delegation->lock); if (delegation->inode != NULL && !test_bit(NFS_DELEGATION_REVOKED, &delegation->flags) &&
@@ -1577,3 +1588,18 @@ bool nfs4_delegation_flush_on_close(const struct inode *inode)rcu_read_unlock ();return ret; } + +int nfs4_delegation_hash_alloc (struct nfs_server *server) +{ +int delegation_buckets, i; + + delegation_buckets =roundup_pow_of_two (nfs_delegation_watermark /16 ); + server->delegation_hash_mask = delegation_buckets -1 ; + server->delegation_hash_table =kmalloc_array (delegation_buckets, +sizeof (*server->delegation_hash_table),GFP_KERNEL ); +if (!server->delegation_hash_table) +return -ENOMEM ; +for (i =0 ; i < delegation_buckets; i++) +INIT_HLIST_HEAD (&server->delegation_hash_table[i]); +return 0 ; +}
fs/nfs/delegation.h +3 -0
@@ -14,6 +14,7 @@* NFSv4 delegation */ struct nfs_delegation { +struct hlist_node hash;struct list_head super_list;const struct cred *cred;struct inode *inode;
@@ -123,4 +124,6 @@ static inline int nfs_have_delegated_mtime(struct inode *inode) NFS_DELEGATION_FLAG_TIME); } +int nfs4_delegation_hash_alloc (struct nfs_server *server); +#endif
fs/nfs/nfs4client.c +5 -0
@@ -802,6 +802,7 @@ static void nfs4_destroy_server(struct nfs_server *server)unset_pnfs_layoutdriver (server);nfs4_purge_state_owners (server, &freeme);nfs4_free_state_owners (&freeme); +kfree (server->delegation_hash_table); }/*
@@ -1096,6 +1097,10 @@ static int nfs4_server_common_setup(struct nfs_server *server, {int error; + error =nfs4_delegation_hash_alloc (server); +if (error) +return error; +/* data servers support only a subset of NFSv4.1 */ if (is_ds_only_client (server->nfs_client))return -EPROTONOSUPPORT ;
fs/nfs/nfs4proc.c +21 -1
@@ -10967,6 +10967,26 @@ static const struct inode_operations nfs4_file_inode_operations = { .listxattr = nfs4_listxattr, }; +static struct nfs_server *nfs4_clone_server (struct nfs_server *source, +struct nfs_fh *fh,struct nfs_fattr *fattr, +rpc_authflavor_t flavor) +{ +struct nfs_server *server; +int error; + + server =nfs_clone_server (source, fh, fattr, flavor); +if (IS_ERR (server)) +return server; + + error =nfs4_delegation_hash_alloc (server); +if (error) { +nfs_free_server (server); +return ERR_PTR (error); + } + +return server; +} +const struct nfs_rpc_ops nfs_v4_clientops = { .version =4 ,/* protocol version */ .dentry_ops = &nfs4_dentry_operations,
@@ -11019,7 +11039,7 @@ const struct nfs_rpc_ops nfs_v4_clientops = { .init_client = nfs4_init_client, .free_client = nfs4_free_client, .create_server = nfs4_create_server, - .clone_server = nfs_clone_server, + .clone_server = nfs4_clone_server, .discover_trunking = nfs4_discover_trunking, .enable_swap = nfs4_enable_swap, .disable_swap = nfs4_disable_swap,
include/linux/nfs_fs_sb.h +2 -0
@@ -255,6 +255,8 @@ struct nfs_server {struct list_head layouts;struct list_head delegations;atomic_long_t nr_active_delegations; +unsigned int delegation_hash_mask; +struct hlist_head *delegation_hash_table;struct list_head ss_copies;struct list_head ss_src_copies;
Signature · 1 line
2.47.2