knfsd: split svc_serv into pools
Subsystems (3)
PATCH 1/1 knfsd: split svc_serv into pools
knfsd: Split out the list of idle threads and pending sockets from svc_serv into a new svc_pool structure, and allocate a fixed number (in this patch, 1) of pools per svc_serv. The new structure contains a lock which takes over several of the duties of svc_serv->sv_lock, which is now relegated to protecting only sv_tempsocks, sv_permsocks, and sv_tmpcnt in svc_serv.
The point is to move the hottest fields out of svc_serv and into svc_pool, allowing a following patch to arrange for a svc_pool per
NUMA node or per CPU. This is a major step towards making the NFS
server NUMA-friendly.
Signed-off-by Greg Banks <[email protected]>
---
include/linux/sunrpc/svc.h | 26 +++++- include/linux/sunrpc/svcsock.h | 1 net/sunrpc/svc.c | 48 ++++++++++- net/sunrpc/svcsock.c | 123 +++++++++++++++++++----------- 4 files changed, 146 insertions(+), 52 deletions(-)
include/linux/sunrpc/svc.h +22 -2
@@ -17,6 +17,25 @@#include <linux/wait.h> #include <linux/mm.h> + +/* +* +* RPC service thread pool. +* +* Pool of threads and temporary sockets. Generally there is only +* a single one of these per RPC service, but on NUMA machines those +* services that can benefit from it (i.e. nfs but not lockd) will +* have one pool per NUMA node. This optimisation reduces cross- +* node traffic on multi-node NUMA NFS servers. +*/ +struct svc_pool { +unsigned int sp_id;/* pool id; also node id on NUMA */ +spinlock_t sp_lock;/* protects all fields */ +struct list_head sp_threads;/* idle server threads */ +struct list_head sp_sockets;/* pending sockets */ +unsigned int sp_nrthreads;/* # of threads in pool */ +}____cacheline_aligned_in_smp ; +/* * RPC service. *
@@ -28,8 +47,6 @@* We currently do not support more than one RPC program per daemon. */ struct svc_serv { -struct list_head sv_threads;/* idle server threads */ -struct list_head sv_sockets;/* pending sockets */ struct svc_program * sv_program;/* RPC program */ struct svc_stat * sv_stats;/* RPC statistics */ spinlock_t sv_lock;
@@ -43,6 +60,8 @@ struct svc_serv {struct timer_list sv_temptimer;/* timer for aging temporary sockets */ char * sv_name;/* service name */ +unsigned int sv_nrpools;/* number of thread pools */ +struct svc_pool * sv_pools;/* array of thread pools */ };/*
@@ -116,6 +135,7 @@ struct svc_rqst {int rq_addrlen;struct svc_serv * rq_server;/* RPC service definition */ +struct svc_pool * rq_pool;/* thread pool */ struct svc_procedure * rq_procinfo;/* procedure info */ struct auth_ops * rq_authop;/* authentication flavour */ struct svc_cred rq_cred;/* auth info */
include/linux/sunrpc/svcsock.h +1 -0
@@ -20,6 +20,7 @@ struct svc_sock {struct socket * sk_sock;/* berkeley socket layer */ struct sock * sk_sk;/* INET layer */ +struct svc_pool * sk_pool;/* current pool iff queued */ struct svc_serv * sk_server;/* service for this socket */ atomic_t sk_inuse;/* use count */ unsigned long sk_flags;
net/sunrpc/svc.c +47 -6
106 lines not on this page · raw message
net/sunrpc/svcsock.c +79 -44
281 lines not on this page · raw message
Signature · 13 lines
Greg Banks, R&D Software Engineer, SGI Australian Software Group. I don't speak for SGI. ------------------------------------------------------------------------- Take Surveys. Earn Cash. Influence the Future of IT Join SourceForge.net's Techsay panel and you'll get the chance to share your opinions on IT & business topics through brief surveys -- and earn cash http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV _______________________________________________ NFS maillist - [email protected] https://lists.sourceforge.net/lists/listinfo/nfs