nfs/sunrpc: high CPU usage in rpcauth_lookup_credcache with lots of uids [PATCH]
Subsystems (3)
[repost, with more people in the cc: list]
We run a cluster of POP and IMAP servers that are used by hundreds of thousands of different users- we see as much as 80.000 different users per server on any given day.
The servers have their mailspool NFS mounted, and each user has a unique user-id.
We noticed a high CPU load on these servers, sometimes near 100% system time being used on all cores. I used oprofile to find out where the CPU was being burned and it turned out to be in rpcauth_lookup_credcache(). The credential cache hashtable is sized way to small for our purposes- it's just 16 slots, and with 100.000 entries in the generic and unix credential caches the chainlength goes over 6000.
It's not that we have tens of thousands of different uids active on the box at the same time, it's just that the box has lots of RAM and all the caches - including the credential caches - just grow and grow.
It appears that both the auth_unix and auth_generic credential caches grow at the same rate and are the same size, but most of the entries in the auth_generic credcache are inactive and could be pruned by a rpcauth_prune_expired() run, where the auth_unix credcache contains mostly active (refcounted) entries. The auth_unix credcache can only be pruned by echo 3>/proc/sys/vm/drop_caches.
The patch below adds a hashtable size module parameter which if set to e.g. 4096 fixes the problem for us. Note that this is a patch for 2.6.27.x, it applies almost cleanly to 2.6.29.3, but it's a proof-of-concept patch anyway. I'm not sure if this is the right or complete fix, we should look into things like:
- is there a way to limit the size of the active number
of entries in the auth_unix credential cache ?
- should the hashtables be size initially based on the amount of RAM ?
- should the hashtables be resized dynamically during use ?
- should rpcauth_cache_shrinker be run more often, not only when there is memory pressure but also from say a timer interrupt ? What should be used for nr_to_scan in that case ?
- why are both auth_generic and auth_unix used (probably has a good reason but I don't know much of the rpc/auth stuff really)
The simple fix would be to size the caches based on some RAM-based heuristic, VFS pressure will then limit the hash chainlength to something reasonable. Correct ? What would that heuristic look like ?
# ------ #
Detailed patch description:
1. bugfixes:
- rpcauth_prune_expired: check nr_to_scan == 0 before decrementing it (in 2.6.29.3 nr_to_scan is decremented but otherwise ignored ?..)
- rpcauth_prune_expired: keep cred on unused list when
cred->cr_count == 0 and it's not expired yet
(also fixed in 2.6.29.3) - set RPC_AUTH_EXPIRY_MORATORIUM to 5*HZ to make it match with the comment ('5 second garbage collection moratorium') (in 2.6.29.3 the comment has been fixed instead)
2. hashtable sizing:
rpcauth_init_credcache: allocate hashtables dyamically auth_unix.c: use rpcauth_init_credcache auth_generic.c: use rpcauth_init_credcache sunrpc_syms.c: add hashsize module parameter
init_sunrpc: call rpcauth_init_module with hashsize parameter
3. debugging:
- rpcauth_prune_expired: don't always delete the
entire cred_unused list, active entries are retained.
rpcauth_lookup_credcache: add new creds to cred_unused right away, making the cred_unused list really a cred_lru list of all creds
- adds sysctl debugging to analyze the number of active and inactive entries on the cred_unused list
- add a sysctl knob to read the longest chainlength reached on any hashtable
include/linux/sunrpc/auth.h renamed from linux-2.6.27.21/include/linux/sunrpc/auth.h +7 -4
@@ -62,8 +62,12 @@*/ #define RPC_CREDCACHE_HASHBITS 4#define RPC_CREDCACHE_NR (1 << RPC_CREDCACHE_HASHBITS) +#define RPC_CREDCACHE_MIN 4 +#define RPC_CREDCACHE_MAX 16384struct rpc_cred_cache { -struct hlist_head hashtable[RPC_CREDCACHE_NR ]; +int hashsize; +int hashbits; +struct hlist_head *hashtable;spinlock_t lock; };
@@ -124,9 +128,8 @@extern const struct rpc_authops authunix_ops;extern const struct rpc_authops authnull_ops; -void __init rpc_init_authunix (void ); -void __init rpc_init_generic_auth (void ); -void __init rpcauth_init_module (void ); +int __init rpc_init_generic_auth (void ); +int __init rpcauth_init_module (int ); void __exit rpcauth_remove_module(void); void __exit rpc_destroy_generic_auth(void);
net/sunrpc/auth.c renamed from linux-2.6.27.21/net/sunrpc/auth.c +174 -17
283 lines not on this page · raw message
net/sunrpc/auth_generic.c renamed from linux-2.6.27.21/net/sunrpc/auth_generic.c +3 -9
@@ -26,7 +26,6 @@ };static struct rpc_auth generic_auth; -static struct rpc_cred_cache generic_cred_cache;static const struct rpc_credops generic_credops;/*
@@ -158,20 +157,16 @@return 0 ; } -void __init rpc_init_generic_auth (void ) +int __init rpc_init_generic_auth (void ) { -spin_lock_init (&generic_cred_cache.lock); +return rpcauth_init_credcache (&generic_auth); } void __exit rpc_destroy_generic_auth(void) { -rpcauth_clear_credcache (&generic_cred_cache); +rpcauth_destroy_credcache (&generic_auth); } -static struct rpc_cred_cache generic_cred_cache = { - {{NULL , },}, -}; -static const struct rpc_authops generic_auth_ops = { .owner =THIS_MODULE , .au_name ="Generic" ,
@@ -182,7 +177,6 @@static struct rpc_auth generic_auth = { .au_ops = &generic_auth_ops, .au_count =ATOMIC_INIT (0 ), - .au_credcache = &generic_cred_cache, };static const struct rpc_credops generic_credops = {
net/sunrpc/auth_unix.c renamed from linux-2.6.27.21/net/sunrpc/auth_unix.c +9 -11
@@ -28,15 +28,23 @@#endif static struct rpc_auth unix_auth; -static struct rpc_cred_cache unix_cred_cache;static const struct rpc_credops unix_credops;static struct rpc_auth *unx_create (struct rpc_clnt *clnt,rpc_authflavor_t flavor) { +int err; +dprintk ("RPC: creating UNIX authenticator for client %p\n" , clnt);atomic_inc (&unix_auth.au_count); +if (!unix_auth.au_credcache) { + err =rpcauth_init_credcache (&unix_auth); +if (err) { +atomic_dec (&unix_auth.au_count); +return ERR_PTR (err); + } + }return &unix_auth; }
@@ -202,11 +210,6 @@return p; } -void __init rpc_init_authunix (void ) -{ -spin_lock_init (&unix_cred_cache.lock); -} -const struct rpc_authops authunix_ops = { .owner =THIS_MODULE , .au_flavor =RPC_AUTH_UNIX ,
@@ -218,17 +221,12 @@ };static -struct rpc_cred_cache unix_cred_cache = { -}; - -static struct rpc_auth unix_auth = { .au_cslack =UNX_WRITESLACK , .au_rslack =2 ,/* assume AUTH_NULL verf */ .au_ops = &authunix_ops, .au_flavor =RPC_AUTH_UNIX , .au_count =ATOMIC_INIT (0 ), - .au_credcache = &unix_cred_cache, }; static
net/sunrpc/sunrpc_syms.c renamed from linux-2.6.27.21/net/sunrpc/sunrpc_syms.c +16 -5
@@ -23,6 +23,7 @@#include <linux/sunrpc/xprtsock.h> extern struct cache_detail ip_map_cache, unix_gid_cache; +static int hashsize;static int __init init_sunrpc (void )
@@ -31,13 +32,14 @@if (err) goto out; err =rpc_init_mempool (); -if (err) { -unregister_rpc_pipefs (); - goto out; - } +if (err) + goto out_err1;#ifdef RPC_DEBUG rpc_register_sysctl ();#endif + err =rpcauth_init_module (hashsize); +if (err) + goto out_err2;#ifdef CONFIG_PROC_FS rpc_proc_init ();#endif
@@ -45,7 +47,14 @@cache_register (&unix_gid_cache);svc_init_xprt_sock ();/* svc sock transport */ init_socket_xprt ();/* clnt sock transport */ -rpcauth_init_module (); + goto out; +out_err2: +rpc_destroy_mempool (); +#ifdef RPC_DEBUG +rpc_unregister_sysctl (); +#endif +out_err1: +unregister_rpc_pipefs (); out:return err; }
@@ -67,6 +76,8 @@rpc_proc_exit ();#endif } +module_param (hashsize, int,0 ); +MODULE_PARM_DESC (hashsize,"size of hashtables for credential caches" );MODULE_LICENSE ("GPL" );module_init (init_sunrpc);module_exit (cleanup_sunrpc);
Quoted 6 lines · Miquel van Smoorenburg
[repost, with more people in the cc: list]
We run a cluster of POP and IMAP servers that are used by hundreds of thousands of different users- we see as much as 80.000 different users per server on any given day.
Quoted 7 lines
We noticed a high CPU load on these servers, sometimes near 100% system time being used on all cores. I used oprofile to find out where the CPU was being burned and it turned out to be in
rpcauth_lookup_credcache(). The credential cache hashtable is sized way to small for our purposes- it's just 16 slots, and with 100.000 entries in the generic and unix credential caches the chainlength goes over 6000.
ping ?
Mike.