nfsd/sunrpc: convert nfsstat server-side interfaces to use netlink
Subsystems (4)
Jeff Layton
maintainerv3 cover nfsd/sunrpc: convert nfsstat server-side interfaces to use netlink
The nfsstat tool currently scrapes /proc/net/rpc/nfsd for server statistics. This procfs interface has several limitations: the counters are global (not network-namespace-aware), the format is fragile to parse, and it cannot be extended without breaking existing parsers.
This series adds per-network-namespace procedure call counts to the sunrpc layer and exposes them through a new netlink handler in the nfsd generic netlink family, allowing nfsstat to retrieve server statistics via netlink with a procfs fallback for older kernels.
Patch 1 adds per-netns per-procedure percpu call count arrays to struct svc_stat, allocated alongside the nfsd_net lifecycle.
Patch 2 switches svc_seq_show() to read from the per-netns counters, making /proc/net/rpc/nfsd namespace-aware. Note that this is a behavior change, but I think it's a desirable one.
Patch 3 implements the server-stats-get netlink dump handler, streaming reply-cache, filehandle, IO, network, and RPC statistics plus per-version and per-operation procedure counts.
Patch 4 removes the now-unused global svc_version vs_count percpu arrays from nfsd, lockd, and the NFS client callback service.
I'll also be sending an nfs-utils patch to convert nfsstat to use the new interfaces.
---
Changes in v3:
- Only increment per-net stats for the "primary" program
- Link to v2: https://lore.kernel.org/r/[email protected]
---
Jeff Layton (4):
sunrpc: add per-netns per-procedure call counts to svc_stat
sunrpc: use per-net counts in svc_seq_show()
nfsd: implement server-stats-get netlink handler
sunrpc: remove unused svc_version vs_count field
Documentation/netlink/specs/nfsd.yaml | 105 ++++++++++++++++ fs/lockd/svc4proc.c | 4 - fs/lockd/svcproc.c | 7 -- fs/nfs/callback_xdr.c | 6 - fs/nfsd/localio.c | 3 - fs/nfsd/netlink.c | 5 + fs/nfsd/netlink.h | 2 + fs/nfsd/nfs2acl.c | 3 - fs/nfsd/nfs3acl.c | 3 - fs/nfsd/nfs3proc.c | 3 - fs/nfsd/nfs4proc.c | 3 - fs/nfsd/nfsctl.c | 221 +++++++++++++++++++++++++++++++++- fs/nfsd/nfsproc.c | 3 - include/linux/sunrpc/stats.h | 6 + include/linux/sunrpc/svc.h | 1 - include/uapi/linux/nfsd_netlink.h | 35 ++++++ net/sunrpc/stats.c | 56 ++++++++- net/sunrpc/svc.c | 9 +- 18 files changed, 435 insertions(+), 40 deletions(-)
---
base-commit: dd886cc1628e04a21a34016635b2b833916a1003
change-id: 20260316-exportd-netlink-1c9fb52536e3
Best regards,
Signature · 1 line
Jeff Layton <[email protected]>
Jeff Layton
maintainerPATCH v3 1/4 sunrpc: add per-netns per-procedure call counts to svc_stat
The existing per-procedure call counts live in global svc_version->vs_count[] arrays which are not network-namespace-aware. Add per-netns equivalents in struct svc_stat so the upcoming netlink stats interface can return namespace-scoped statistics.
Add a vs_count pointer array to struct svc_stat, along with svc_stat_alloc_counts() and svc_stat_free_counts() helpers to manage per-version percpu call count arrays.
Increment the per-net counter alongside the global one in svc_generic_init_request(). Call the alloc/free helpers from nfsd_net_init() and nfsd_net_exit().
--- fs/nfsd/nfsctl.c | 8 ++++++- include/linux/sunrpc/stats.h | 6 +++++ net/sunrpc/stats.c | 54 ++++++++++++++++++++++++++++++++++++++++++++ net/sunrpc/svc.c | 8 +++++++ 4 files changed, 75 insertions(+), 1 deletion(-)
fs/nfsd/nfsctl.c +7 -1
@@ -2436,9 +2436,12 @@ static __net_init int nfsd_net_init(struct net *net)memset (&nn->nfsd_svcstats,0 ,sizeof (nn->nfsd_svcstats)); nn->nfsd_svcstats.program = &nfsd_programs[0 ]; + retval =svc_stat_alloc_counts (&nn->nfsd_svcstats); +if (retval) + goto out_proc_error;if (!nfsd_proc_stat_init (net)) { retval = -ENOMEM ; - goto out_proc_error; + goto out_svcstats_error; }for (i =0 ; i <sizeof (nn->nfsd_versions); i++)
@@ -2456,6 +2459,8 @@ static __net_init int nfsd_net_init(struct net *net)#endif return 0 ; +out_svcstats_error: +svc_stat_free_counts (&nn->nfsd_svcstats); out_proc_error:percpu_counter_destroy_many (nn->counter,NFSD_STATS_COUNTERS_NUM ); out_repcache_error:
@@ -2496,6 +2501,7 @@ static __net_exit void nfsd_net_exit(struct net *net)kfree_sensitive (nn->fh_key);nfsd_net_cb_shutdown (nn);nfsd_proc_stat_shutdown (net); +svc_stat_free_counts (&nn->nfsd_svcstats);percpu_counter_destroy_many (nn->counter,NFSD_STATS_COUNTERS_NUM );nfsd_idmap_shutdown (net);nfsd_export_shutdown (net);
include/linux/sunrpc/stats.h +6 -0
@@ -37,9 +37,15 @@ struct svc_stat { rpcbadfmt, rpcbadauth, rpcbadclnt; + +/* Per-version per-procedure call counts (per-cpu, per-netns) */ +unsigned long __percpu **vs_count; };struct net ; +int svc_stat_alloc_counts (struct svc_stat *statp); +void svc_stat_free_counts (struct svc_stat *statp); +#ifdef CONFIG_PROC_FS int rpc_proc_init (struct net *);void rpc_proc_exit (struct net *);
net/sunrpc/stats.c +54 -0
60 lines not on this page · raw message
net/sunrpc/svc.c +8 -0
@@ -1408,6 +1408,14 @@ svc_generic_init_request(struct svc_rqst *rqstp,/* Bump per-procedure stats counter */ this_cpu_inc (versp->vs_count[rqstp->rq_proc]); +/* Bump per-net per-procedure stats counter */ +if (rqstp->rq_server->sv_stats && + rqstp->rq_server->sv_stats->program == progp && + rqstp->rq_server->sv_stats->vs_count && + rqstp->rq_server->sv_stats->vs_count[rqstp->rq_vers]) +this_cpu_inc (rqstp->rq_server->sv_stats->vs_count + [rqstp->rq_vers][rqstp->rq_proc]); + ret->dispatch = versp->vs_dispatch;return rpc_success; err_bad_vers:
Signature · 1 line
2.54.0
Jeff Layton
maintainerPATCH v3 2/4 sunrpc: use per-net counts in svc_seq_show()
In reply to Jeff LaytonUpdate svc_seq_show() to read from the per-netns statp->vs_count[] arrays instead of the global svc_version->vs_count[].
The only caller is nfsd, which always allocates vs_count via svc_stat_alloc_counts() in nfsd_net_init(), so the per-netns arrays are always available.
This makes /proc/net/rpc/nfsd report per-network-namespace procedure call counts.
--- net/sunrpc/stats.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
net/sunrpc/stats.c +1 -1
@@ -108,7 +108,7 @@ void svc_seq_show(struct seq_file *seq, const struct svc_stat *statp)for (j =0 ; j < vers->vs_nproc; j++) { count =0 ;for_each_possible_cpu (k) - count +=per_cpu (vers->vs_count[j], k); + count +=per_cpu (statp->vs_count[i][j], k);seq_printf (seq," %lu" , count); }seq_putc (seq,'\n' );
Signature · 1 line
2.54.0
Jeff Layton
maintainerPATCH v3 3/4 nfsd: implement server-stats-get netlink handler
In reply to Jeff LaytonImplement nfsd_nl_server_stats_get_dumpit() which exposes the NFS server statistics currently available via /proc/net/rpc/nfsd through the nfsd generic netlink family.
The handler uses a dump operation to stream statistics across multiple netlink messages:
- First message: all scalar stats (reply cache, filehandle, IO, network, RPC) plus per-version procedure counts (proc2/3/4-ops) using per-netns vs_count arrays.
- Subsequent messages: NFSv4 per-operation counts (proc4ops-ops), one entry per message, using cb->args[0] to track the current operation index across dump calls.
This allows nfsstat to retrieve server statistics via netlink with a procfs fallback for older kernels.
--- Documentation/netlink/specs/nfsd.yaml | 105 +++++++++++++++++ fs/nfsd/netlink.c | 5 + fs/nfsd/netlink.h | 2 + fs/nfsd/nfsctl.c | 213 ++++++++++++++++++++++++++++++++++ include/uapi/linux/nfsd_netlink.h | 35 ++++++ 5 files changed, 360 insertions(+)
Documentation/netlink/specs/nfsd.yaml +105 -0
@@ -330,6 +330,86 @@ attribute-sets: of which client holds the state. Intended for use after all clients have been unexported from a given path, enabling the underlying filesystem to be unmounted. + - + name: server-proc-entry + attributes: + - + name: op + type: u32 + - + name: count + type: u64 + - + name: pad + type: pad + - + name: server-stats + attributes: + - + name: rc-hits + type: u64 + - + name: rc-misses + type: u64 + - + name: rc-nocache + type: u64 + - + name: pad + type: pad + - + name: fh-stale + type: u64 + - + name: io-read + type: u64 + - + name: io-write + type: u64 + - + name: netcnt + type: u32 + - + name: netudpcnt + type: u32 + - + name: nettcpcnt + type: u32 + - + name: nettcpconn + type: u32 + - + name: rpccnt + type: u32 + - + name: rpcbadfmt + type: u32 + - + name: rpcbadauth + type: u32 + - + name: rpcbadclnt + type: u32 + - + name: proc2-ops + type: nest + nested-attributes: server-proc-entry + multi-attr: true + - + name: proc3-ops + type: nest + nested-attributes: server-proc-entry + multi-attr: true + - + name: proc4-ops + type: nest + nested-attributes: server-proc-entry + multi-attr: true + - + name: proc4ops-ops + type: nest + nested-attributes: server-proc-entry + multi-attr: true operations: list:
@@ -516,6 +596,31 @@ operations: request: attributes: - path + - + name: server-stats-get + doc: dump NFS server statistics + attribute-set: server-stats + dump: + reply: + attributes: + - rc-hits + - rc-misses + - rc-nocache + - fh-stale + - io-read + - io-write + - netcnt + - netudpcnt + - nettcpcnt + - nettcpconn + - rpccnt + - rpcbadfmt + - rpcbadauth + - rpcbadclnt + - proc2-ops + - proc3-ops + - proc4-ops + - proc4ops-ops mcast-groups: list:
fs/nfsd/netlink.c +5 -0
@@ -225,6 +225,11 @@ static const struct genl_split_ops nfsd_nl_ops[] = { .maxattr = NFSD_A_UNLOCK_EXPORT_PATH, .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, }, + { + .cmd = NFSD_CMD_SERVER_STATS_GET, + .dumpit = nfsd_nl_server_stats_get_dumpit, + .flags = GENL_CMD_CAP_DUMP, + }, };static const struct genl_multicast_group nfsd_nl_mcgrps[] = {
fs/nfsd/netlink.h +2 -0
@@ -42,6 +42,8 @@ int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info);int nfsd_nl_unlock_ip_doit (struct sk_buff *skb,struct genl_info *info);int nfsd_nl_unlock_filesystem_doit (struct sk_buff *skb,struct genl_info *info);int nfsd_nl_unlock_export_doit (struct sk_buff *skb,struct genl_info *info); +int nfsd_nl_server_stats_get_dumpit (struct sk_buff *skb, +struct netlink_callback *cb);enum {NFSD_NLGRP_NONE ,
fs/nfsd/nfsctl.c +213 -0
219 lines not on this page · raw message
include/uapi/linux/nfsd_netlink.h +35 -0
@@ -225,6 +225,40 @@ enum {NFSD_A_UNLOCK_EXPORT_MAX = (__NFSD_A_UNLOCK_EXPORT_MAX -1 ) }; +enum { +NFSD_A_SERVER_PROC_ENTRY_OP =1 , +NFSD_A_SERVER_PROC_ENTRY_COUNT , +NFSD_A_SERVER_PROC_ENTRY_PAD , + + __NFSD_A_SERVER_PROC_ENTRY_MAX, +NFSD_A_SERVER_PROC_ENTRY_MAX = (__NFSD_A_SERVER_PROC_ENTRY_MAX -1 ) +}; + +enum { +NFSD_A_SERVER_STATS_RC_HITS =1 , +NFSD_A_SERVER_STATS_RC_MISSES , +NFSD_A_SERVER_STATS_RC_NOCACHE , +NFSD_A_SERVER_STATS_PAD , +NFSD_A_SERVER_STATS_FH_STALE , +NFSD_A_SERVER_STATS_IO_READ , +NFSD_A_SERVER_STATS_IO_WRITE , +NFSD_A_SERVER_STATS_NETCNT , +NFSD_A_SERVER_STATS_NETUDPCNT , +NFSD_A_SERVER_STATS_NETTCPCNT , +NFSD_A_SERVER_STATS_NETTCPCONN , +NFSD_A_SERVER_STATS_RPCCNT , +NFSD_A_SERVER_STATS_RPCBADFMT , +NFSD_A_SERVER_STATS_RPCBADAUTH , +NFSD_A_SERVER_STATS_RPCBADCLNT , +NFSD_A_SERVER_STATS_PROC2_OPS , +NFSD_A_SERVER_STATS_PROC3_OPS , +NFSD_A_SERVER_STATS_PROC4_OPS , +NFSD_A_SERVER_STATS_PROC4OPS_OPS , + + __NFSD_A_SERVER_STATS_MAX, +NFSD_A_SERVER_STATS_MAX = (__NFSD_A_SERVER_STATS_MAX -1 ) +}; +enum {NFSD_CMD_RPC_STATUS_GET =1 ,NFSD_CMD_THREADS_SET ,
@@ -244,6 +278,7 @@ enum {NFSD_CMD_UNLOCK_IP ,NFSD_CMD_UNLOCK_FILESYSTEM ,NFSD_CMD_UNLOCK_EXPORT , +NFSD_CMD_SERVER_STATS_GET , __NFSD_CMD_MAX,NFSD_CMD_MAX = (__NFSD_CMD_MAX -1 )
Signature · 1 line
2.54.0
Jeff Layton
maintainerPATCH v3 4/4 sunrpc: remove unused svc_version vs_count field
In reply to Jeff LaytonNow that svc_seq_show() and the nfsd netlink stats handler both use the per-netns svc_stat vs_count arrays, the global per-version vs_count percpu counters are no longer read by anything. Remove the vs_count field from struct svc_version and all the associated DEFINE_PER_CPU_ALIGNED arrays and initializers across nfsd, lockd, and the NFS client callback service.
--- fs/lockd/svc4proc.c | 4 ---- fs/lockd/svcproc.c | 7 ------- fs/nfs/callback_xdr.c | 6 ------ fs/nfsd/localio.c | 3 --- fs/nfsd/nfs2acl.c | 3 --- fs/nfsd/nfs3acl.c | 3 --- fs/nfsd/nfs3proc.c | 3 --- fs/nfsd/nfs4proc.c | 3 --- fs/nfsd/nfsproc.c | 3 --- include/linux/sunrpc/svc.h | 1 - net/sunrpc/svc.c | 3 --- 11 files changed, 39 deletions(-)
fs/lockd/svc4proc.c +0 -4
@@ -1420,14 +1420,10 @@ union nlm4svc_xdrstore {struct nlm4_shareres_wrapper shareres; }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nlm4svc_call_counters[ARRAY_SIZE(nlm4svc_procedures)]); -const struct svc_version nlmsvc_version4 = { .vs_vers =4 , .vs_nproc =ARRAY_SIZE (nlm4svc_procedures), .vs_proc = nlm4svc_procedures, - .vs_count = nlm4svc_call_counters, .vs_dispatch = nlmsvc_dispatch, .vs_xdrsize =sizeof (union nlm4svc_xdrstore ), };
fs/lockd/svcproc.c +0 -7
@@ -1433,25 +1433,18 @@ union nlmsvc_xdrstore {* NLMv1 defines only procedures 1 - 15. Linux lockd also implements * procedures 0 (NULL) and 16 (SM_NOTIFY). */ -static DEFINE_PER_CPU_ALIGNED(unsigned long, nlm1svc_call_counters[17]); -const struct svc_version nlmsvc_version1 = { .vs_vers =1 , .vs_nproc =17 , .vs_proc = nlmsvc_procedures, - .vs_count = nlm1svc_call_counters, .vs_dispatch = nlmsvc_dispatch, .vs_xdrsize =sizeof (union nlmsvc_xdrstore ), }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nlm3svc_call_counters[ARRAY_SIZE(nlmsvc_procedures)]); -const struct svc_version nlmsvc_version3 = { .vs_vers =3 , .vs_nproc =ARRAY_SIZE (nlmsvc_procedures), .vs_proc = nlmsvc_procedures, - .vs_count = nlm3svc_call_counters, .vs_dispatch = nlmsvc_dispatch, .vs_xdrsize =sizeof (union nlmsvc_xdrstore ), };
fs/nfs/callback_xdr.c +0 -6
@@ -1090,26 +1090,20 @@ static const struct svc_procedure nfs4_callback_procedures1[] = { } }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfs4_callback_count1[ARRAY_SIZE(nfs4_callback_procedures1)]);const struct svc_version nfs4_callback_version1 = { .vs_vers =1 , .vs_nproc =ARRAY_SIZE (nfs4_callback_procedures1), .vs_proc = nfs4_callback_procedures1, - .vs_count = nfs4_callback_count1, .vs_xdrsize =NFS4_CALLBACK_XDRSIZE , .vs_dispatch = nfs_callback_dispatch, .vs_hidden = true, .vs_need_cong_ctrl = true, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfs4_callback_count4[ARRAY_SIZE(nfs4_callback_procedures1)]);const struct svc_version nfs4_callback_version4 = { .vs_vers =4 , .vs_nproc =ARRAY_SIZE (nfs4_callback_procedures1), .vs_proc = nfs4_callback_procedures1, - .vs_count = nfs4_callback_count4, .vs_xdrsize =NFS4_CALLBACK_XDRSIZE , .vs_dispatch = nfs_callback_dispatch, .vs_hidden = true,
fs/nfsd/localio.c +0 -3
@@ -210,14 +210,11 @@ static const struct svc_procedure localio_procedures1[] = { };#define LOCALIO_NR_PROCEDURES ARRAY_SIZE(localio_procedures1) -static DEFINE_PER_CPU_ALIGNED(unsigned long, - localio_count[LOCALIO_NR_PROCEDURES]);const struct svc_version localio_version1 = { .vs_vers =1 , .vs_nproc =LOCALIO_NR_PROCEDURES , .vs_proc = localio_procedures1, .vs_dispatch = nfsd_dispatch, - .vs_count = localio_count, .vs_xdrsize =XDR_QUADLEN (UUID_SIZE ), .vs_hidden = true, };
fs/nfsd/nfs2acl.c +0 -3
@@ -389,13 +389,10 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { }, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfsd_acl_count2[ARRAY_SIZE(nfsd_acl_procedures2)]);const struct svc_version nfsd_acl_version2 = { .vs_vers =2 , .vs_nproc =ARRAY_SIZE (nfsd_acl_procedures2), .vs_proc = nfsd_acl_procedures2, - .vs_count = nfsd_acl_count2, .vs_dispatch = nfsd_dispatch, .vs_xdrsize =NFS3_SVC_XDRSIZE , };
fs/nfsd/nfs3acl.c +0 -3
@@ -278,13 +278,10 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = { }, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfsd_acl_count3[ARRAY_SIZE(nfsd_acl_procedures3)]);const struct svc_version nfsd_acl_version3 = { .vs_vers =3 , .vs_nproc =ARRAY_SIZE (nfsd_acl_procedures3), .vs_proc = nfsd_acl_procedures3, - .vs_count = nfsd_acl_count3, .vs_dispatch = nfsd_dispatch, .vs_xdrsize =NFS3_SVC_XDRSIZE , };
fs/nfsd/nfs3proc.c +0 -3
@@ -1068,13 +1068,10 @@ static const struct svc_procedure nfsd_procedures3[22] = { }, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfsd_count3[ARRAY_SIZE(nfsd_procedures3)]);const struct svc_version nfsd_version3 = { .vs_vers =3 , .vs_nproc =ARRAY_SIZE (nfsd_procedures3), .vs_proc = nfsd_procedures3, .vs_dispatch = nfsd_dispatch, - .vs_count = nfsd_count3, .vs_xdrsize =NFS3_SVC_XDRSIZE , };
fs/nfsd/nfs4proc.c +0 -3
@@ -4129,13 +4129,10 @@ static const struct svc_procedure nfsd_procedures4[2] = { }, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfsd_count4[ARRAY_SIZE(nfsd_procedures4)]);const struct svc_version nfsd_version4 = { .vs_vers =4 , .vs_nproc =ARRAY_SIZE (nfsd_procedures4), .vs_proc = nfsd_procedures4, - .vs_count = nfsd_count4, .vs_dispatch = nfsd_dispatch, .vs_xdrsize =NFS4_SVC_XDRSIZE , .vs_rpcb_optnl = true,
fs/nfsd/nfsproc.c +0 -3
@@ -838,13 +838,10 @@ static const struct svc_procedure nfsd_procedures2[18] = { }, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfsd_count2[ARRAY_SIZE(nfsd_procedures2)]);const struct svc_version nfsd_version2 = { .vs_vers =2 , .vs_nproc =ARRAY_SIZE (nfsd_procedures2), .vs_proc = nfsd_procedures2, - .vs_count = nfsd_count2, .vs_dispatch = nfsd_dispatch, .vs_xdrsize =NFS2_SVC_XDRSIZE , };
include/linux/sunrpc/svc.h +0 -1
@@ -408,7 +408,6 @@ struct svc_version {u32 vs_vers;/* version number */ u32 vs_nproc;/* number of procedures */ const struct svc_procedure *vs_proc;/* per-procedure info */ -unsigned long __percpu *vs_count;/* call counts */ u32 vs_xdrsize;/* xdrsize needed for this version */ /* Don't register with rpcbind */
net/sunrpc/svc.c +0 -3
@@ -1405,9 +1405,6 @@ svc_generic_init_request(struct svc_rqst *rqstp,memset (rqstp->rq_argp,0 , procp->pc_argzero);memset (rqstp->rq_resp,0 , procp->pc_ressize); -/* Bump per-procedure stats counter */ -this_cpu_inc (versp->vs_count[rqstp->rq_proc]); -/* Bump per-net per-procedure stats counter */ if (rqstp->rq_server->sv_stats && rqstp->rq_server->sv_stats->program == progp &&
Signature · 1 line
2.54.0