nfsd/sunrpc: convert nfsstat server-side interfaces to use netlink
Subsystems (4)
Jeff Layton
maintainerv6 cover nfsd/sunrpc: convert nfsstat server-side interfaces to use netlink
This version fixes a few problems that Chuck and Sashiko pointed out in review.
The nfsstat tool currently scrapes /proc/net/rpc/nfsd for server statistics. This procfs interface has several limitations: the counters are global (not network-namespace-aware), the format is fragile to parse, and it cannot be extended without breaking existing parsers.
This series adds per-network-namespace procedure call counts to the sunrpc layer and exposes them through a new netlink handler in the nfsd generic netlink family, allowing nfsstat to retrieve server statistics via netlink with a procfs fallback for older kernels. The nfs-utils patch for that part is already merged.
Additionally, this version adds tracking of callback operation counts.
---
Changes in v6:
- Emit the entire server-stats object in a single netlink dump message
- Count CB_SEQUENCE on v4.1+ callbacks, for parity with the forechannel SEQUENCE count
- Fix a use-after-free in nfsd4_run_cb() by snapshotting the opcode, net, and minorversion
- Link to v5: https://lore.kernel.org/r/[email protected]
Changes in v5:
- Add NFSv4 callback call counts to nfsstat upcall
- Link to v4: https://lore.kernel.org/r/[email protected]
Changes in v4:
- Ensure error is returned when nla_put_*() calls fail
- Link to v3: https://lore.kernel.org/r/[email protected]
Changes in v3:
- Only increment per-net stats for the "primary" program
- Link to v2: https://lore.kernel.org/r/[email protected]
---
Jeff Layton (6):
sunrpc: add per-netns per-procedure call counts to svc_stat
sunrpc: use per-net counts in svc_seq_show()
nfsd: implement server-stats-get netlink handler
sunrpc: remove unused svc_version vs_count field
nfsd: count NFSv4 callback operations per netns
nfsd: export NFSv4 callback op stats via netlink
Documentation/netlink/specs/nfsd.yaml | 111 +++++++++++++++++ fs/lockd/svc4proc.c | 4 - fs/lockd/svcproc.c | 7 -- fs/nfs/callback_xdr.c | 6 - fs/nfsd/localio.c | 3 - fs/nfsd/netlink.c | 5 + fs/nfsd/netlink.h | 2 + fs/nfsd/netns.h | 12 +- fs/nfsd/nfs2acl.c | 3 - fs/nfsd/nfs3acl.c | 3 - fs/nfsd/nfs3proc.c | 3 - fs/nfsd/nfs4callback.c | 22 +++- fs/nfsd/nfs4proc.c | 3 - fs/nfsd/nfs4state.c | 2 - fs/nfsd/nfsctl.c | 222 +++++++++++++++++++++++++++++++++- fs/nfsd/nfsproc.c | 3 - fs/nfsd/stats.c | 2 +- fs/nfsd/stats.h | 5 +- include/linux/sunrpc/stats.h | 6 + include/linux/sunrpc/svc.h | 1 - include/uapi/linux/nfsd_netlink.h | 36 ++++++ net/sunrpc/stats.c | 2 +- net/sunrpc/svc.c | 63 +++++++++- 23 files changed, 479 insertions(+), 47 deletions(-)
---
base-commit: 6f90c7618528b5ca5887f8c6057f26dcc7a27a99
change-id: 20260316-exportd-netlink-1c9fb52536e3
Best regards,
Signature · 1 line
Jeff Layton <[email protected]>
Jeff Layton
maintainerPATCH v6 1/6 sunrpc: add per-netns per-procedure call counts to svc_stat
The existing per-procedure call counts live in global svc_version->vs_count[] arrays which are not network-namespace-aware. Add per-netns equivalents in struct svc_stat so the upcoming netlink stats interface can return namespace-scoped statistics.
Add a vs_count pointer array to struct svc_stat, along with svc_stat_alloc_counts() and svc_stat_free_counts() helpers to manage per-version percpu call count arrays.
Increment the per-net counter alongside the global one in svc_generic_init_request(). Call the alloc/free helpers from nfsd_net_init() and nfsd_net_exit().
--- fs/nfsd/nfsctl.c | 8 +++++- include/linux/sunrpc/stats.h | 6 +++++ net/sunrpc/svc.c | 62 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-)
fs/nfsd/nfsctl.c +7 -1
@@ -2514,9 +2514,12 @@ static __net_init int nfsd_net_init(struct net *net)memset (&nn->nfsd_svcstats,0 ,sizeof (nn->nfsd_svcstats)); nn->nfsd_svcstats.program = &nfsd_programs[0 ]; + retval =svc_stat_alloc_counts (&nn->nfsd_svcstats); +if (retval) + goto out_proc_error;if (!nfsd_proc_stat_init (net)) { retval = -ENOMEM ; - goto out_proc_error; + goto out_svcstats_error; }for (i =0 ; i <sizeof (nn->nfsd_versions); i++)
@@ -2534,6 +2537,8 @@ static __net_init int nfsd_net_init(struct net *net)#endif return 0 ; +out_svcstats_error: +svc_stat_free_counts (&nn->nfsd_svcstats); out_proc_error:percpu_counter_destroy_many (nn->counter,NFSD_STATS_COUNTERS_NUM ); out_repcache_error:
@@ -2574,6 +2579,7 @@ static __net_exit void nfsd_net_exit(struct net *net)kfree_sensitive (nn->fh_key);nfsd_net_cb_shutdown (nn);nfsd_proc_stat_shutdown (net); +svc_stat_free_counts (&nn->nfsd_svcstats);percpu_counter_destroy_many (nn->counter,NFSD_STATS_COUNTERS_NUM );nfsd_idmap_shutdown (net);nfsd_export_shutdown (net);
include/linux/sunrpc/stats.h +6 -0
@@ -37,9 +37,15 @@ struct svc_stat { rpcbadfmt, rpcbadauth, rpcbadclnt; + +/* Per-version per-procedure call counts (per-cpu, per-netns) */ +unsigned long __percpu **vs_count; };struct net ; +int svc_stat_alloc_counts (struct svc_stat *statp); +void svc_stat_free_counts (struct svc_stat *statp); +#ifdef CONFIG_PROC_FS int rpc_proc_init (struct net *);void rpc_proc_exit (struct net *);
net/sunrpc/svc.c +62 -0
74 lines not on this page · raw message
Signature · 1 line
2.54.0
Jeff Layton
maintainerPATCH v6 2/6 sunrpc: use per-net counts in svc_seq_show()
In reply to Jeff LaytonUpdate svc_seq_show() to read from the per-netns statp->vs_count[] arrays instead of the global svc_version->vs_count[].
The only caller is nfsd, which always allocates vs_count via svc_stat_alloc_counts() in nfsd_net_init(), so the per-netns arrays are always available.
This makes /proc/net/rpc/nfsd report per-network-namespace procedure call counts.
--- net/sunrpc/stats.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
net/sunrpc/stats.c +1 -1
@@ -108,7 +108,7 @@ void svc_seq_show(struct seq_file *seq, const struct svc_stat *statp)for (j =0 ; j < vers->vs_nproc; j++) { count =0 ;for_each_possible_cpu (k) - count +=per_cpu (vers->vs_count[j], k); + count +=per_cpu (statp->vs_count[i][j], k);seq_printf (seq," %lu" , count); }seq_putc (seq,'\n' );
Signature · 1 line
2.54.0
Jeff Layton
maintainerPATCH v6 3/6 nfsd: implement server-stats-get netlink handler
In reply to Jeff LaytonImplement nfsd_nl_server_stats_get_dumpit() which exposes the NFS server statistics currently available via /proc/net/rpc/nfsd through the nfsd generic netlink family.
The handler uses a dump operation to stream statistics across multiple netlink messages:
- First message: all scalar stats (reply cache, filehandle, IO, network, RPC) plus per-version procedure counts (proc2/3/4-ops) using per-netns vs_count arrays.
- Subsequent messages: NFSv4 per-operation counts (proc4ops-ops), one entry per message, using cb->args[0] to track the current operation index across dump calls.
This allows nfsstat to retrieve server statistics via netlink with a procfs fallback for older kernels.
--- Documentation/netlink/specs/nfsd.yaml | 105 ++++++++++++++++++++ fs/nfsd/netlink.c | 5 + fs/nfsd/netlink.h | 2 + fs/nfsd/nfsctl.c | 179 ++++++++++++++++++++++++++++++++++ include/uapi/linux/nfsd_netlink.h | 35 +++++++ 5 files changed, 326 insertions(+)
Documentation/netlink/specs/nfsd.yaml +105 -0
@@ -330,6 +330,86 @@ attribute-sets: of which client holds the state. Intended for use after all clients have been unexported from a given path, enabling the underlying filesystem to be unmounted. + - + name: server-proc-entry + attributes: + - + name: op + type: u32 + - + name: count + type: u64 + - + name: pad + type: pad + - + name: server-stats + attributes: + - + name: rc-hits + type: u64 + - + name: rc-misses + type: u64 + - + name: rc-nocache + type: u64 + - + name: pad + type: pad + - + name: fh-stale + type: u64 + - + name: io-read + type: u64 + - + name: io-write + type: u64 + - + name: netcnt + type: u32 + - + name: netudpcnt + type: u32 + - + name: nettcpcnt + type: u32 + - + name: nettcpconn + type: u32 + - + name: rpccnt + type: u32 + - + name: rpcbadfmt + type: u32 + - + name: rpcbadauth + type: u32 + - + name: rpcbadclnt + type: u32 + - + name: proc2-ops + type: nest + nested-attributes: server-proc-entry + multi-attr: true + - + name: proc3-ops + type: nest + nested-attributes: server-proc-entry + multi-attr: true + - + name: proc4-ops + type: nest + nested-attributes: server-proc-entry + multi-attr: true + - + name: proc4ops-ops + type: nest + nested-attributes: server-proc-entry + multi-attr: true operations: list:
@@ -516,6 +596,31 @@ operations: request: attributes: - path + - + name: server-stats-get + doc: dump NFS server statistics + attribute-set: server-stats + dump: + reply: + attributes: + - rc-hits + - rc-misses + - rc-nocache + - fh-stale + - io-read + - io-write + - netcnt + - netudpcnt + - nettcpcnt + - nettcpconn + - rpccnt + - rpcbadfmt + - rpcbadauth + - rpcbadclnt + - proc2-ops + - proc3-ops + - proc4-ops + - proc4ops-ops mcast-groups: list:
fs/nfsd/netlink.c +5 -0
@@ -225,6 +225,11 @@ static const struct genl_split_ops nfsd_nl_ops[] = { .maxattr = NFSD_A_UNLOCK_EXPORT_PATH, .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, }, + { + .cmd = NFSD_CMD_SERVER_STATS_GET, + .dumpit = nfsd_nl_server_stats_get_dumpit, + .flags = GENL_CMD_CAP_DUMP, + }, };static const struct genl_multicast_group nfsd_nl_mcgrps[] = {
fs/nfsd/netlink.h +2 -0
@@ -42,6 +42,8 @@ int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info);int nfsd_nl_unlock_ip_doit (struct sk_buff *skb,struct genl_info *info);int nfsd_nl_unlock_filesystem_doit (struct sk_buff *skb,struct genl_info *info);int nfsd_nl_unlock_export_doit (struct sk_buff *skb,struct genl_info *info); +int nfsd_nl_server_stats_get_dumpit (struct sk_buff *skb, +struct netlink_callback *cb);enum {NFSD_NLGRP_NONE ,
fs/nfsd/nfsctl.c +179 -0
185 lines not on this page · raw message
include/uapi/linux/nfsd_netlink.h +35 -0
@@ -225,6 +225,40 @@ enum {NFSD_A_UNLOCK_EXPORT_MAX = (__NFSD_A_UNLOCK_EXPORT_MAX -1 ) }; +enum { +NFSD_A_SERVER_PROC_ENTRY_OP =1 , +NFSD_A_SERVER_PROC_ENTRY_COUNT , +NFSD_A_SERVER_PROC_ENTRY_PAD , + + __NFSD_A_SERVER_PROC_ENTRY_MAX, +NFSD_A_SERVER_PROC_ENTRY_MAX = (__NFSD_A_SERVER_PROC_ENTRY_MAX -1 ) +}; + +enum { +NFSD_A_SERVER_STATS_RC_HITS =1 , +NFSD_A_SERVER_STATS_RC_MISSES , +NFSD_A_SERVER_STATS_RC_NOCACHE , +NFSD_A_SERVER_STATS_PAD , +NFSD_A_SERVER_STATS_FH_STALE , +NFSD_A_SERVER_STATS_IO_READ , +NFSD_A_SERVER_STATS_IO_WRITE , +NFSD_A_SERVER_STATS_NETCNT , +NFSD_A_SERVER_STATS_NETUDPCNT , +NFSD_A_SERVER_STATS_NETTCPCNT , +NFSD_A_SERVER_STATS_NETTCPCONN , +NFSD_A_SERVER_STATS_RPCCNT , +NFSD_A_SERVER_STATS_RPCBADFMT , +NFSD_A_SERVER_STATS_RPCBADAUTH , +NFSD_A_SERVER_STATS_RPCBADCLNT , +NFSD_A_SERVER_STATS_PROC2_OPS , +NFSD_A_SERVER_STATS_PROC3_OPS , +NFSD_A_SERVER_STATS_PROC4_OPS , +NFSD_A_SERVER_STATS_PROC4OPS_OPS , + + __NFSD_A_SERVER_STATS_MAX, +NFSD_A_SERVER_STATS_MAX = (__NFSD_A_SERVER_STATS_MAX -1 ) +}; +enum {NFSD_CMD_RPC_STATUS_GET =1 ,NFSD_CMD_THREADS_SET ,
@@ -244,6 +278,7 @@ enum {NFSD_CMD_UNLOCK_IP ,NFSD_CMD_UNLOCK_FILESYSTEM ,NFSD_CMD_UNLOCK_EXPORT , +NFSD_CMD_SERVER_STATS_GET , __NFSD_CMD_MAX,NFSD_CMD_MAX = (__NFSD_CMD_MAX -1 )
Signature · 1 line
2.54.0
Chuck Lever
maintainerRe: [PATCH v6 3/6] nfsd: implement server-stats-get netlink handler
Quoted 3 lines · Jeff Layton
Implement
nfsd_nl_server_stats_get_dumpit()which exposes the NFS server statistics currently available via/proc/net/rpc/nfsdthrough the nfsd generic netlink family.
A couple of questions on the dump design and one on the commit message, then some smaller items.
The commit message says:
Quoted 7 lines
- First message: all scalar stats (reply cache, filehandle, IO, network, RPC) plus per-version procedure counts (proc2/3/4-ops) using per-netns
vs_countarrays.- Subsequent messages: NFSv4 per-operation counts (proc4ops-ops), one entry per message, using cb->args[0] to track the current operation index across dump calls.
The code does not stream across subsequent messages. The proc4ops-ops nests are emitted in the first message, inside the same "if (start == 0)" block as the scalars, and cb->args[0] only ever holds 0 or -1, never an operation index. The function's own kerneldoc agrees: "The entire server-stats object is emitted in a single netlink message on the first invocation."
Should the commit message be reworded to match the code, or should the code move to the streaming design the message describes? That second option ties into the next question.
Quoted 62 lines
fs/nfsd/nfsctl.c +54
-0@@ -2329,6 +2329,185 @@ int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info)return 0 ; } +static int nfsd_nl_fill_proc_ops (struct sk_buff *skb,int attr, +unsigned long __percpu *counts, +unsigned int nproc) +{ +struct nlattr *nest; +unsigned int j; +int k; + +for (j =0 ; j < nproc; j++) { +unsigned long count =0 ; + +for_each_possible_cpu (k) + count +=per_cpu (counts[j], k); + + nest =nla_nest_start (skb, attr); +if (!nest) +return -EMSGSIZE ; +if (nla_put_u32 (skb,NFSD_A_SERVER_PROC_ENTRY_OP , j) || +nla_put_u64_64bit (skb,NFSD_A_SERVER_PROC_ENTRY_COUNT , + count,NFSD_A_SERVER_PROC_ENTRY_PAD )) { +nla_nest_cancel (skb, nest); +return -EMSGSIZE ; + } +nla_nest_end (skb, nest); + } + +return 0 ; +} + +/** +* nfsd_nl_server_stats_get_dumpit - dump NFS server statistics +* @skb: reply buffer +* @cb: netlink metadata and command arguments +* +* The entire server-stats object is emitted in a single netlink message on +* the first invocation. cb->args[0] is set to -1 afterwards so that the next +* invocation terminates the dump. +* +* Returns the size of the reply or a negative errno. +*/ +int nfsd_nl_server_stats_get_dumpit (struct sk_buff *skb, +struct netlink_callback *cb) +{ +struct net *net =sock_net (skb->sk); +struct nfsd_net *nn =net_generic (net, nfsd_net_id); +struct svc_stat *statp = &nn->nfsd_svcstats; +struct svc_program *prog = statp->program; +int start = cb->args[0 ]; +void *hdr; + +/* +* cb->args[0] == 0: first call, emit the full server-stats object +* cb->args[0] < 0: dump already complete +*/
fs/nfsd/nfsctl.c int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info) · line 2395
Quoted 10 lines
fs/nfsd/nfsctl.c +10
-0+if (start <0 ) +return 0 ; + +if (start ==0 ) { + hdr =genlmsg_put (skb,NETLINK_CB (cb->skb).portid, + cb->nlh->nlmsg_seq, &nfsd_nl_family, +NLM_F_MULTI ,NFSD_CMD_SERVER_STATS_GET ); +if (!hdr) +return -ENOBUFS ; +
This handler assembles the entire server-stats object -- all scalars, the proc2/3/4-ops nests, and one proc4ops-ops nest per NFSv4 operation -- in one message under "if (start == 0)", then sets cb->args[0] = -1 so the next call ends the dump.
Does that hold up once the assembled message no longer fits the dump buffer? A dump skb is allocated at max(cb->min_dump_alloc, NLMSG_GOODSIZE), and this handler sets neither cb->min_dump_alloc nor a .start callback, so the floor is NLMSG_GOODSIZE, which is SKB_WITH_OVERHEAD(PAGE_SIZE), roughly 3.7k on a 4k-page host. A larger skb is allocated only when userspace has previously grown its receive buffer.
With LAST_NFS4_OP at 75, the proc4ops-ops loop alone emits 76 nests of about 28 bytes each (~2.1k); the proc2 and proc3 nests plus the scalar block bring the total to roughly 3.5k -- already within a few hundred bytes of the floor, and growing by one nest for every NFSv4 operation added later.
[ ... scalar puts for reply cache, fh, io, net, rpc snipped ... ]
Quoted 36 lines
fs/nfsd/nfsctl.c +36
-0+/* Per-version procedure counts */ +if (statp->vs_count) { +static const int proc_attrs[] = { + [2 ] =NFSD_A_SERVER_STATS_PROC2_OPS , + [3 ] =NFSD_A_SERVER_STATS_PROC3_OPS , + [4 ] =NFSD_A_SERVER_STATS_PROC4_OPS , + }; +unsigned int i; + +for (i =0 ; i < prog->pg_nvers && + i <ARRAY_SIZE (proc_attrs); i++) { +if (!prog->pg_vers[i] || + !statp->vs_count[i]) +continue ; +if (!proc_attrs[i]) +continue ; +if (nfsd_nl_fill_proc_ops (skb, + proc_attrs[i], + statp->vs_count[i], + prog->pg_vers[i]->vs_nproc)) + goto err_cancel; + } + } + +#ifdef CONFIG_NFSD_V4 +/* NFSv4 individual operation counts */ +for (int i =0 ; i <=LAST_NFS4_OP ; i++) { +struct nlattr *nest; +u64 cnt; + + cnt =percpu_counter_sum_positive ( + &nn->counter[NFSD_STATS_NFS4_OP (i)]); + + nest =nla_nest_start (skb, +NFSD_A_SERVER_STATS_PROC4OPS_OPS ); +if (!nest)
fs/nfsd/nfsctl.c int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info) · line 2498
Quoted 10 lines
fs/nfsd/nfsctl.c +10
-0+ goto err_cancel; +if (nla_put_u32 (skb,NFSD_A_SERVER_PROC_ENTRY_OP , i) || +nla_put_u64_64bit (skb,NFSD_A_SERVER_PROC_ENTRY_COUNT , + cnt,NFSD_A_SERVER_PROC_ENTRY_PAD )) { +nla_nest_cancel (skb, nest); + goto err_cancel; + } + nla_nest_end(skb, nest); + } +#endif
This loop open-codes the same nest that nfsd_nl_fill_proc_ops() builds just above -- nla_nest_start(), nla_put_u32(PROC_ENTRY_OP), nla_put_u64_64bit(PROC_ENTRY_COUNT), nla_nest_end() -- into the same NFSD_A_SERVER_STATS_PROC4OPS_OPS attribute. Could the helper be generalized to take the per-op counter source so this is not a second copy of the same code?
The per-version block above skips empty versions:
if (!prog->pg_vers[i] || !statp->vs_count[i]) continue;
but this loop emits an entry for every op 0..LAST_NFS4_OP, zero-count ops included. Is that difference intentional? Skipping zero counts here would also trim the worst-case message size above.
There is also a counter that this dump does not emit. /proc/net/rpc/nfsd prints a wdeleg_getattr line after proc4ops:
seq_printf(seq, "\nwdeleg_getattr %lld", percpu_counter_sum_positive(&nn->counter[NFSD_STATS_WDELEG_GETATTR]));
incremented by nfsd_stats_wdeleg_getattr_inc(). Since the goal is to expose the statistics currently available via /proc/net/rpc/nfsd, should wdeleg_getattr get an attribute here too, so nfsstat over netlink does not drop it relative to the procfs path?
fs/nfsd/nfsctl.c int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info) · line 2509
Quoted 11 lines
fs/nfsd/nfsctl.c +11
-0+ +genlmsg_end (skb, hdr); + } + + cb->args[0 ] =-1 ; +return skb->len; + +err_cancel: +genlmsg_cancel (skb, hdr); +return -EMSGSIZE ; +}
Back to the buffer question: when an nla_put eventually fails, control reaches err_cancel, and genlmsg_cancel() trims the partial message back out, so skb->len is 0 on return. In netlink_dump():
if (nlk->dump_done_errno == -EMSGSIZE && skb->len) nlk->dump_done_errno = skb->len;
the skb->len test fails, so -EMSGSIZE is delivered to userspace as a terminal error rather than being taken as "buffer full, call again". With everything emitted under "if (start == 0)" and no per-attribute cursor in cb->args, a retry rebuilds the same oversized message, so the dump cannot ever complete.
nfsd_nl_rpc_status_get_dumpit() already follows the streaming pattern the commit message describes: one entry per message, saving the cursor in cb->args and returning skb->len on EMSGSIZE. Would building this dump the same way be more robust than the single-message form?
Signature · 1 line
Chuck Lever
Jeff Layton
maintainerRe: [PATCH v6 3/6] nfsd: implement server-stats-get netlink handler
Quoted 188 lines · Chuck Lever
Quoted 3 lines · Jeff Layton
Implement
nfsd_nl_server_stats_get_dumpit()which exposes the NFS server statistics currently available via/proc/net/rpc/nfsdthrough the nfsd generic netlink family.A couple of questions on the dump design and one on the commit message, then some smaller items.
The commit message says:
Quoted 7 lines
- First message: all scalar stats (reply cache, filehandle, IO, network, RPC) plus per-version procedure counts (proc2/3/4-ops) using per-netns
vs_countarrays.- Subsequent messages: NFSv4 per-operation counts (proc4ops-ops), one entry per message, using cb->args[0] to track the current operation index across dump calls.
The code does not stream across subsequent messages. The proc4ops-ops nests are emitted in the first message, inside the same "if (start == 0)" block as the scalars, and cb->args[0] only ever holds 0 or -1, never an operation index. The function's own kerneldoc agrees: "The entire server-stats object is emitted in a single netlink message on the first invocation."
Should the commit message be reworded to match the code, or should the code move to the streaming design the message describes? That second option ties into the next question.
Quoted 62 lines
fs/nfsd/nfsctl.c +54
-0@@ -2329,6 +2329,185 @@ int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info)return 0 ; } +static int nfsd_nl_fill_proc_ops (struct sk_buff *skb,int attr, +unsigned long __percpu *counts, +unsigned int nproc) +{ +struct nlattr *nest; +unsigned int j; +int k; + +for (j =0 ; j < nproc; j++) { +unsigned long count =0 ; + +for_each_possible_cpu (k) + count +=per_cpu (counts[j], k); + + nest =nla_nest_start (skb, attr); +if (!nest) +return -EMSGSIZE ; +if (nla_put_u32 (skb,NFSD_A_SERVER_PROC_ENTRY_OP , j) || +nla_put_u64_64bit (skb,NFSD_A_SERVER_PROC_ENTRY_COUNT , + count,NFSD_A_SERVER_PROC_ENTRY_PAD )) { +nla_nest_cancel (skb, nest); +return -EMSGSIZE ; + } +nla_nest_end (skb, nest); + } + +return 0 ; +} + +/** +* nfsd_nl_server_stats_get_dumpit - dump NFS server statistics +* @skb: reply buffer +* @cb: netlink metadata and command arguments +* +* The entire server-stats object is emitted in a single netlink message on +* the first invocation. cb->args[0] is set to -1 afterwards so that the next +* invocation terminates the dump. +* +* Returns the size of the reply or a negative errno. +*/ +int nfsd_nl_server_stats_get_dumpit (struct sk_buff *skb, +struct netlink_callback *cb) +{ +struct net *net =sock_net (skb->sk); +struct nfsd_net *nn =net_generic (net, nfsd_net_id); +struct svc_stat *statp = &nn->nfsd_svcstats; +struct svc_program *prog = statp->program; +int start = cb->args[0 ]; +void *hdr; + +/* +* cb->args[0] == 0: first call, emit the full server-stats object +* cb->args[0] < 0: dump already complete +*/ fs/nfsd/nfsctl.c int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info) · line 2395
Quoted 10 lines
fs/nfsd/nfsctl.c +10
-0+if (start <0 ) +return 0 ; + +if (start ==0 ) { + hdr =genlmsg_put (skb,NETLINK_CB (cb->skb).portid, + cb->nlh->nlmsg_seq, &nfsd_nl_family, +NLM_F_MULTI ,NFSD_CMD_SERVER_STATS_GET ); +if (!hdr) +return -ENOBUFS ; +This handler assembles the entire server-stats object -- all scalars, the proc2/3/4-ops nests, and one proc4ops-ops nest per NFSv4 operation -- in one message under "if (start == 0)", then sets cb->args[0] = -1 so the next call ends the dump.
Does that hold up once the assembled message no longer fits the dump buffer? A dump skb is allocated at
max(cb->min_dump_alloc, NLMSG_GOODSIZE), and this handler sets neithercb->min_dump_allocnor a .start callback, so the floor isNLMSG_GOODSIZE, which isSKB_WITH_OVERHEAD(PAGE_SIZE), roughly 3.7k on a 4k-page host. A larger skb is allocated only when userspace has previously grown its receive buffer.With
LAST_NFS4_OPat 75, the proc4ops-ops loop alone emits 76 nests of about 28 bytes each (~2.1k); the proc2 and proc3 nests plus the scalar block bring the total to roughly 3.5k -- already within a few hundred bytes of the floor, and growing by one nest for every NFSv4 operation added later.[ ... scalar puts for reply cache, fh, io, net, rpc snipped ... ]
Quoted 36 lines
fs/nfsd/nfsctl.c +36
-0+/* Per-version procedure counts */ +if (statp->vs_count) { +static const int proc_attrs[] = { + [2 ] =NFSD_A_SERVER_STATS_PROC2_OPS , + [3 ] =NFSD_A_SERVER_STATS_PROC3_OPS , + [4 ] =NFSD_A_SERVER_STATS_PROC4_OPS , + }; +unsigned int i; + +for (i =0 ; i < prog->pg_nvers && + i <ARRAY_SIZE (proc_attrs); i++) { +if (!prog->pg_vers[i] || + !statp->vs_count[i]) +continue ; +if (!proc_attrs[i]) +continue ; +if (nfsd_nl_fill_proc_ops (skb, + proc_attrs[i], + statp->vs_count[i], + prog->pg_vers[i]->vs_nproc)) + goto err_cancel; + } + } + +#ifdef CONFIG_NFSD_V4 +/* NFSv4 individual operation counts */ +for (int i =0 ; i <=LAST_NFS4_OP ; i++) { +struct nlattr *nest; +u64 cnt; + + cnt =percpu_counter_sum_positive ( + &nn->counter[NFSD_STATS_NFS4_OP (i)]); + + nest =nla_nest_start (skb, +NFSD_A_SERVER_STATS_PROC4OPS_OPS ); +if (!nest)fs/nfsd/nfsctl.c int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info) · line 2498
Quoted 10 lines
fs/nfsd/nfsctl.c +10
-0+ goto err_cancel; +if (nla_put_u32 (skb,NFSD_A_SERVER_PROC_ENTRY_OP , i) || +nla_put_u64_64bit (skb,NFSD_A_SERVER_PROC_ENTRY_COUNT , + cnt,NFSD_A_SERVER_PROC_ENTRY_PAD )) { +nla_nest_cancel (skb, nest); + goto err_cancel; + } + nla_nest_end(skb, nest); + } +#endif This loop open-codes the same nest that
nfsd_nl_fill_proc_ops()builds just above --nla_nest_start(),nla_put_u32(PROC_ENTRY_OP),nla_put_u64_64bit(PROC_ENTRY_COUNT),nla_nest_end()-- into the sameNFSD_A_SERVER_STATS_PROC4OPS_OPSattribute. Could the helper be generalized to take the per-op counter source so this is not a second copy of the same code?The per-version block above skips empty versions:
if (!prog->pg_vers[i] || !statp->vs_count[i]) continue;but this loop emits an entry for every op 0..LAST_NFS4_OP, zero-count ops included. Is that difference intentional? Skipping zero counts here would also trim the worst-case message size above.
There is also a counter that this dump does not emit.
/proc/net/rpc/nfsdprints awdeleg_getattrline after proc4ops:
Quoted 8 lines
seq_printf(seq, "\nwdeleg_getattr %lld", percpu_counter_sum_positive(&nn->counter[NFSD_STATS_WDELEG_GETATTR]));incremented by
nfsd_stats_wdeleg_getattr_inc(). Since the goal is to expose the statistics currently available via/proc/net/rpc/nfsd, shouldwdeleg_getattrget an attribute here too, so nfsstat over netlink does not drop it relative to the procfs path?
The only thing in this review that is not addressed in my current series is the above comment, and I think we had agreed in an earlier email thread that this was approximately the same value as the counter for CB_GETATTR calls and so it's good enough for this purpose.
Are you still ok with this approach?
Signature · 1 line
Jeff Layton <[email protected]>
Chuck Lever
maintainerRe: [PATCH v6 3/6] nfsd: implement server-stats-get netlink handler
Quoted 79 lines · Jeff Layton
Quoted 69 lines · Chuck Lever
Quoted 36 lines · Jeff Layton
fs/nfsd/nfsctl.c +36
-0+/* Per-version procedure counts */ +if (statp->vs_count) { +static const int proc_attrs[] = { + [2 ] =NFSD_A_SERVER_STATS_PROC2_OPS , + [3 ] =NFSD_A_SERVER_STATS_PROC3_OPS , + [4 ] =NFSD_A_SERVER_STATS_PROC4_OPS , + }; +unsigned int i; + +for (i =0 ; i < prog->pg_nvers && + i <ARRAY_SIZE (proc_attrs); i++) { +if (!prog->pg_vers[i] || + !statp->vs_count[i]) +continue ; +if (!proc_attrs[i]) +continue ; +if (nfsd_nl_fill_proc_ops (skb, + proc_attrs[i], + statp->vs_count[i], + prog->pg_vers[i]->vs_nproc)) + goto err_cancel; + } + } + +#ifdef CONFIG_NFSD_V4 +/* NFSv4 individual operation counts */ +for (int i =0 ; i <=LAST_NFS4_OP ; i++) { +struct nlattr *nest; +u64 cnt; + + cnt =percpu_counter_sum_positive ( + &nn->counter[NFSD_STATS_NFS4_OP (i)]); + + nest =nla_nest_start (skb, +NFSD_A_SERVER_STATS_PROC4OPS_OPS ); +if (!nest)fs/nfsd/nfsctl.c int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info) · line 2498
Quoted 10 lines
fs/nfsd/nfsctl.c +10
-0+ goto err_cancel; +if (nla_put_u32 (skb,NFSD_A_SERVER_PROC_ENTRY_OP , i) || +nla_put_u64_64bit (skb,NFSD_A_SERVER_PROC_ENTRY_COUNT , + cnt,NFSD_A_SERVER_PROC_ENTRY_PAD )) { +nla_nest_cancel (skb, nest); + goto err_cancel; + } + nla_nest_end(skb, nest); + } +#endif This loop open-codes the same nest that
nfsd_nl_fill_proc_ops()builds just above --nla_nest_start(),nla_put_u32(PROC_ENTRY_OP),nla_put_u64_64bit(PROC_ENTRY_COUNT),nla_nest_end()-- into the sameNFSD_A_SERVER_STATS_PROC4OPS_OPSattribute. Could the helper be generalized to take the per-op counter source so this is not a second copy of the same code?The per-version block above skips empty versions:
if (!prog->pg_vers[i] || !statp->vs_count[i]) continue;but this loop emits an entry for every op 0..LAST_NFS4_OP, zero-count ops included. Is that difference intentional? Skipping zero counts here would also trim the worst-case message size above.
There is also a counter that this dump does not emit.
/proc/net/rpc/nfsdprints awdeleg_getattrline after proc4ops:Quoted 8 lines
seq_printf(seq, "\nwdeleg_getattr %lld", percpu_counter_sum_positive(&nn->counter[NFSD_STATS_WDELEG_GETATTR]));incremented by
nfsd_stats_wdeleg_getattr_inc(). Since the goal is to expose the statistics currently available via/proc/net/rpc/nfsd, shouldwdeleg_getattrget an attribute here too, so nfsstat over netlink does not drop it relative to the procfs path?
Quoted 6 lines
The only thing in this review that is not addressed in my current series is the above comment, and I think we had agreed in an earlier email thread that this was approximately the same value as the counter for
CB_GETATTRcalls and so it's good enough for this purpose.Are you still ok with this approach?
In spite of the email quoting, I’ve still forgotten most of the context of that conversation.
Looking at this now, I think someone later could get an urge to add the wdeleg_getattr statistic to netlink… for symmetry, because it smells like technical debt, or it might actually be useful for something.
IMHO we should include it now.
Signature · 1 line
Chuck Lever
Jeff Layton
maintainerRe: [PATCH v6 3/6] nfsd: implement server-stats-get netlink handler
Quoted 88 lines · Chuck Lever
Quoted 79 lines · Jeff Layton
Quoted 69 lines · Chuck Lever
Quoted 36 lines · Jeff Layton
fs/nfsd/nfsctl.c +36
-0+/* Per-version procedure counts */ +if (statp->vs_count) { +static const int proc_attrs[] = { + [2 ] =NFSD_A_SERVER_STATS_PROC2_OPS , + [3 ] =NFSD_A_SERVER_STATS_PROC3_OPS , + [4 ] =NFSD_A_SERVER_STATS_PROC4_OPS , + }; +unsigned int i; + +for (i =0 ; i < prog->pg_nvers && + i <ARRAY_SIZE (proc_attrs); i++) { +if (!prog->pg_vers[i] || + !statp->vs_count[i]) +continue ; +if (!proc_attrs[i]) +continue ; +if (nfsd_nl_fill_proc_ops (skb, + proc_attrs[i], + statp->vs_count[i], + prog->pg_vers[i]->vs_nproc)) + goto err_cancel; + } + } + +#ifdef CONFIG_NFSD_V4 +/* NFSv4 individual operation counts */ +for (int i =0 ; i <=LAST_NFS4_OP ; i++) { +struct nlattr *nest; +u64 cnt; + + cnt =percpu_counter_sum_positive ( + &nn->counter[NFSD_STATS_NFS4_OP (i)]); + + nest =nla_nest_start (skb, +NFSD_A_SERVER_STATS_PROC4OPS_OPS ); +if (!nest)fs/nfsd/nfsctl.c int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info) · line 2498
Quoted 10 lines
fs/nfsd/nfsctl.c +10
-0+ goto err_cancel; +if (nla_put_u32 (skb,NFSD_A_SERVER_PROC_ENTRY_OP , i) || +nla_put_u64_64bit (skb,NFSD_A_SERVER_PROC_ENTRY_COUNT , + cnt,NFSD_A_SERVER_PROC_ENTRY_PAD )) { +nla_nest_cancel (skb, nest); + goto err_cancel; + } + nla_nest_end(skb, nest); + } +#endif This loop open-codes the same nest that
nfsd_nl_fill_proc_ops()builds just above --nla_nest_start(),nla_put_u32(PROC_ENTRY_OP),nla_put_u64_64bit(PROC_ENTRY_COUNT),nla_nest_end()-- into the sameNFSD_A_SERVER_STATS_PROC4OPS_OPSattribute. Could the helper be generalized to take the per-op counter source so this is not a second copy of the same code?The per-version block above skips empty versions:
if (!prog->pg_vers[i] || !statp->vs_count[i]) continue;but this loop emits an entry for every op 0..LAST_NFS4_OP, zero-count ops included. Is that difference intentional? Skipping zero counts here would also trim the worst-case message size above.
There is also a counter that this dump does not emit.
/proc/net/rpc/nfsdprints awdeleg_getattrline after proc4ops:Quoted 8 lines
seq_printf(seq, "\nwdeleg_getattr %lld", percpu_counter_sum_positive(&nn->counter[NFSD_STATS_WDELEG_GETATTR]));incremented by
nfsd_stats_wdeleg_getattr_inc(). Since the goal is to expose the statistics currently available via/proc/net/rpc/nfsd, shouldwdeleg_getattrget an attribute here too, so nfsstat over netlink does not drop it relative to the procfs path?Quoted 6 lines
The only thing in this review that is not addressed in my current series is the above comment, and I think we had agreed in an earlier email thread that this was approximately the same value as the counter for
CB_GETATTRcalls and so it's good enough for this purpose.Are you still ok with this approach?
Quoted 9 lines
In spite of the email quoting, I’ve still forgotten most of the context of that conversation.
Looking at this now, I think someone later could get an urge to add the
wdeleg_getattrstatistic to netlink… for symmetry, because it smells like technical debt, or it might actually be useful for something.IMHO we should include it now.
IDGI: what's the point of collecting that stat separately when we have stats for CB_GETATTR already? I'd prefer to avoid perpetuating wdeleg_getattr as a stat here, since it's not directly tied to the protocol in the same way as the others are.
The only difference between the two is that if another client does a GETATTR for the fh while there is already a CB_GETATTR in flight, wdeleg_getattr gets bumped today, whereas CB_GETATTR counter does not (since there is only a single CB_GETATTR for that).
Is that difference relevant? I think Dai added wdeleg_getattr originally, so I'd really like to hear his opinion on this.
Signature · 1 line
Jeff Layton <[email protected]>
Chuck Lever
maintainerRe: [PATCH v6 3/6] nfsd: implement server-stats-get netlink handler
Quoted 104 lines · Jeff Layton
Quoted 88 lines · Chuck Lever
Quoted 79 lines · Jeff Layton
Quoted 69 lines · Chuck Lever
Quoted 36 lines · Jeff Layton
fs/nfsd/nfsctl.c +36
-0+/* Per-version procedure counts */ +if (statp->vs_count) { +static const int proc_attrs[] = { + [2 ] =NFSD_A_SERVER_STATS_PROC2_OPS , + [3 ] =NFSD_A_SERVER_STATS_PROC3_OPS , + [4 ] =NFSD_A_SERVER_STATS_PROC4_OPS , + }; +unsigned int i; + +for (i =0 ; i < prog->pg_nvers && + i <ARRAY_SIZE (proc_attrs); i++) { +if (!prog->pg_vers[i] || + !statp->vs_count[i]) +continue ; +if (!proc_attrs[i]) +continue ; +if (nfsd_nl_fill_proc_ops (skb, + proc_attrs[i], + statp->vs_count[i], + prog->pg_vers[i]->vs_nproc)) + goto err_cancel; + } + } + +#ifdef CONFIG_NFSD_V4 +/* NFSv4 individual operation counts */ +for (int i =0 ; i <=LAST_NFS4_OP ; i++) { +struct nlattr *nest; +u64 cnt; + + cnt =percpu_counter_sum_positive ( + &nn->counter[NFSD_STATS_NFS4_OP (i)]); + + nest =nla_nest_start (skb, +NFSD_A_SERVER_STATS_PROC4OPS_OPS ); +if (!nest)fs/nfsd/nfsctl.c int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info) · line 2498
Quoted 10 lines
fs/nfsd/nfsctl.c +10
-0+ goto err_cancel; +if (nla_put_u32 (skb,NFSD_A_SERVER_PROC_ENTRY_OP , i) || +nla_put_u64_64bit (skb,NFSD_A_SERVER_PROC_ENTRY_COUNT , + cnt,NFSD_A_SERVER_PROC_ENTRY_PAD )) { +nla_nest_cancel (skb, nest); + goto err_cancel; + } + nla_nest_end(skb, nest); + } +#endif This loop open-codes the same nest that
nfsd_nl_fill_proc_ops()builds just above --nla_nest_start(),nla_put_u32(PROC_ENTRY_OP),nla_put_u64_64bit(PROC_ENTRY_COUNT),nla_nest_end()-- into the sameNFSD_A_SERVER_STATS_PROC4OPS_OPSattribute. Could the helper be generalized to take the per-op counter source so this is not a second copy of the same code?The per-version block above skips empty versions:
if (!prog->pg_vers[i] || !statp->vs_count[i]) continue;but this loop emits an entry for every op 0..LAST_NFS4_OP, zero-count ops included. Is that difference intentional? Skipping zero counts here would also trim the worst-case message size above.
There is also a counter that this dump does not emit.
/proc/net/rpc/nfsdprints awdeleg_getattrline after proc4ops:Quoted 8 lines
seq_printf(seq, "\nwdeleg_getattr %lld", percpu_counter_sum_positive(&nn->counter[NFSD_STATS_WDELEG_GETATTR]));incremented by
nfsd_stats_wdeleg_getattr_inc(). Since the goal is to expose the statistics currently available via/proc/net/rpc/nfsd, shouldwdeleg_getattrget an attribute here too, so nfsstat over netlink does not drop it relative to the procfs path?Quoted 6 lines
The only thing in this review that is not addressed in my current series is the above comment, and I think we had agreed in an earlier email thread that this was approximately the same value as the counter for
CB_GETATTRcalls and so it's good enough for this purpose.Are you still ok with this approach?
Quoted 9 lines
In spite of the email quoting, I’ve still forgotten most of the context of that conversation.
Looking at this now, I think someone later could get an urge to add the
wdeleg_getattrstatistic to netlink… for symmetry, because it smells like technical debt, or it might actually be useful for something.IMHO we should include it now.
IDGI: what's the point of collecting that stat separately when we have stats for
CB_GETATTRalready? I'd prefer to avoid perpetuatingwdeleg_getattras a stat here, since it's not directly tied to the protocol in the same way as the others are.
Quoted 7 lines
The only difference between the two is that if another client does a GETATTR for the fh while there is already a
CB_GETATTRin flight,wdeleg_getattrgets bumped today, whereasCB_GETATTRcounter does not (since there is only a singleCB_GETATTRfor that).Is that difference relevant? I think Dai added
wdeleg_getattroriginally, so I'd really like to hear his opinion on this.
OK, so you are actually re-litigating whether the stat should exist at all. I confess I had forgotten that context.
I’m arguing only that the netlink and procfs APIs should remain equivalent, at least initially. Deleting the stat entirely probably has no consequences right now, but let’s hear from Dai when the US west coast wakes up.
Signature · 1 line
Chuck Lever
Chuck Lever
maintainerRe: [PATCH v6 3/6] nfsd: implement server-stats-get netlink handler
Quoted 113 lines · Chuck Lever
Quoted 104 lines · Jeff Layton
Quoted 88 lines · Chuck Lever
Quoted 79 lines · Jeff Layton
Quoted 69 lines · Chuck Lever
Quoted 36 lines · Jeff Layton
fs/nfsd/nfsctl.c +36
-0+/* Per-version procedure counts */ +if (statp->vs_count) { +static const int proc_attrs[] = { + [2 ] =NFSD_A_SERVER_STATS_PROC2_OPS , + [3 ] =NFSD_A_SERVER_STATS_PROC3_OPS , + [4 ] =NFSD_A_SERVER_STATS_PROC4_OPS , + }; +unsigned int i; + +for (i =0 ; i < prog->pg_nvers && + i <ARRAY_SIZE (proc_attrs); i++) { +if (!prog->pg_vers[i] || + !statp->vs_count[i]) +continue ; +if (!proc_attrs[i]) +continue ; +if (nfsd_nl_fill_proc_ops (skb, + proc_attrs[i], + statp->vs_count[i], + prog->pg_vers[i]->vs_nproc)) + goto err_cancel; + } + } + +#ifdef CONFIG_NFSD_V4 +/* NFSv4 individual operation counts */ +for (int i =0 ; i <=LAST_NFS4_OP ; i++) { +struct nlattr *nest; +u64 cnt; + + cnt =percpu_counter_sum_positive ( + &nn->counter[NFSD_STATS_NFS4_OP (i)]); + + nest =nla_nest_start (skb, +NFSD_A_SERVER_STATS_PROC4OPS_OPS ); +if (!nest)fs/nfsd/nfsctl.c int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info) · line 2498
Quoted 10 lines
fs/nfsd/nfsctl.c +10
-0+ goto err_cancel; +if (nla_put_u32 (skb,NFSD_A_SERVER_PROC_ENTRY_OP , i) || +nla_put_u64_64bit (skb,NFSD_A_SERVER_PROC_ENTRY_COUNT , + cnt,NFSD_A_SERVER_PROC_ENTRY_PAD )) { +nla_nest_cancel (skb, nest); + goto err_cancel; + } + nla_nest_end(skb, nest); + } +#endif This loop open-codes the same nest that
nfsd_nl_fill_proc_ops()builds just above --nla_nest_start(),nla_put_u32(PROC_ENTRY_OP),nla_put_u64_64bit(PROC_ENTRY_COUNT),nla_nest_end()-- into the sameNFSD_A_SERVER_STATS_PROC4OPS_OPSattribute. Could the helper be generalized to take the per-op counter source so this is not a second copy of the same code?The per-version block above skips empty versions:
if (!prog->pg_vers[i] || !statp->vs_count[i]) continue;but this loop emits an entry for every op 0..LAST_NFS4_OP, zero-count ops included. Is that difference intentional? Skipping zero counts here would also trim the worst-case message size above.
There is also a counter that this dump does not emit.
/proc/net/rpc/nfsdprints awdeleg_getattrline after proc4ops:Quoted 8 lines
seq_printf(seq, "\nwdeleg_getattr %lld", percpu_counter_sum_positive(&nn->counter[NFSD_STATS_WDELEG_GETATTR]));incremented by
nfsd_stats_wdeleg_getattr_inc(). Since the goal is to expose the statistics currently available via/proc/net/rpc/nfsd, shouldwdeleg_getattrget an attribute here too, so nfsstat over netlink does not drop it relative to the procfs path?Quoted 6 lines
The only thing in this review that is not addressed in my current series is the above comment, and I think we had agreed in an earlier email thread that this was approximately the same value as the counter for
CB_GETATTRcalls and so it's good enough for this purpose.Are you still ok with this approach?
Quoted 9 lines
In spite of the email quoting, I’ve still forgotten most of the context of that conversation.
Looking at this now, I think someone later could get an urge to add the
wdeleg_getattrstatistic to netlink… for symmetry, because it smells like technical debt, or it might actually be useful for something.IMHO we should include it now.
IDGI: what's the point of collecting that stat separately when we have stats for
CB_GETATTRalready? I'd prefer to avoid perpetuatingwdeleg_getattras a stat here, since it's not directly tied to the protocol in the same way as the others are.Quoted 7 lines
The only difference between the two is that if another client does a GETATTR for the fh while there is already a
CB_GETATTRin flight,wdeleg_getattrgets bumped today, whereasCB_GETATTRcounter does not (since there is only a singleCB_GETATTRfor that).Is that difference relevant? I think Dai added
wdeleg_getattroriginally, so I'd really like to hear his opinion on this.
Quoted 7 lines
OK, so you are actually re-litigating whether the stat should exist at all. I confess I had forgotten that context.
I’m arguing only that the netlink and procfs APIs should remain equivalent, at least initially. Deleting the stat entirely probably has no consequences right now, but let’s hear from Dai when the US west coast wakes up.
Another option is to keep the stat but report it some other way. It is arguable whether nfsstat is the proper mechanism to report the different reasons for sending a CB_GETATTR.
Signature · 1 line
Chuck Lever
Jeff Layton
maintainerRe: [PATCH v6 3/6] nfsd: implement server-stats-get netlink handler
Quoted 112 lines · Chuck Lever
Quoted 110 lines · Chuck Lever
Quoted 109 lines · Jeff Layton
Quoted 88 lines · Chuck Lever
Quoted 79 lines · Jeff Layton
Quoted 69 lines · Chuck Lever
Quoted 36 lines · Jeff Layton
(quoted diff) +36
-0+ /* Per-version procedure counts */ + if (statp->vs_count) { + static const int proc_attrs[] = { + [2] = NFSD_A_SERVER_STATS_PROC2_OPS, + [3] = NFSD_A_SERVER_STATS_PROC3_OPS, + [4] = NFSD_A_SERVER_STATS_PROC4_OPS, + }; + unsigned int i; + + for (i = 0; i < prog->pg_nvers && + i < ARRAY_SIZE(proc_attrs); i++) { + if (!prog->pg_vers[i] || + !statp->vs_count[i]) + continue; + if (!proc_attrs[i]) + continue; + if (nfsd_nl_fill_proc_ops(skb, + proc_attrs[i], + statp->vs_count[i], + prog->pg_vers[i]->vs_nproc)) + goto err_cancel; + } + } + +#ifdef CONFIG_NFSD_V4 + /* NFSv4 individual operation counts */ + for (int i = 0; i <= LAST_NFS4_OP; i++) { + struct nlattr *nest; + u64 cnt; + + cnt = percpu_counter_sum_positive( + &nn->counter[NFSD_STATS_NFS4_OP(i)]); + + nest = nla_nest_start(skb, + NFSD_A_SERVER_STATS_PROC4OPS_OPS); + if (!nest)Review
Quoted 10 lines
(quoted diff) +10
-0+ goto err_cancel; + if (nla_put_u32(skb, NFSD_A_SERVER_PROC_ENTRY_OP, i) || + nla_put_u64_64bit(skb, NFSD_A_SERVER_PROC_ENTRY_COUNT, + cnt, NFSD_A_SERVER_PROC_ENTRY_PAD)) { + nla_nest_cancel(skb, nest); + goto err_cancel; + } + nla_nest_end(skb, nest); + } +#endifThis loop open-codes the same nest that
nfsd_nl_fill_proc_ops()builds just above --nla_nest_start(),nla_put_u32(PROC_ENTRY_OP),nla_put_u64_64bit(PROC_ENTRY_COUNT),nla_nest_end()-- into the sameNFSD_A_SERVER_STATS_PROC4OPS_OPSattribute. Could the helper be generalized to take the per-op counter source so this is not a second copy of the same code?The per-version block above skips empty versions:
if (!prog->pg_vers[i] || !statp->vs_count[i]) continue;but this loop emits an entry for every op 0..LAST_NFS4_OP, zero-count ops included. Is that difference intentional? Skipping zero counts here would also trim the worst-case message size above.
There is also a counter that this dump does not emit.
/proc/net/rpc/nfsdprints awdeleg_getattrline after proc4ops:Quoted 8 lines
seq_printf(seq, "\nwdeleg_getattr %lld", percpu_counter_sum_positive(&nn->counter[NFSD_STATS_WDELEG_GETATTR]));incremented by
nfsd_stats_wdeleg_getattr_inc(). Since the goal is to expose the statistics currently available via/proc/net/rpc/nfsd, shouldwdeleg_getattrget an attribute here too, so nfsstat over netlink does not drop it relative to the procfs path?Quoted 6 lines
The only thing in this review that is not addressed in my current series is the above comment, and I think we had agreed in an earlier email thread that this was approximately the same value as the counter for
CB_GETATTRcalls and so it's good enough for this purpose.Are you still ok with this approach?
Quoted 9 lines
In spite of the email quoting, I’ve still forgotten most of the context of that conversation.
Looking at this now, I think someone later could get an urge to add the
wdeleg_getattrstatistic to netlink… for symmetry, because it smells like technical debt, or it might actually be useful for something.IMHO we should include it now.
IDGI: what's the point of collecting that stat separately when we have stats for
CB_GETATTRalready? I'd prefer to avoid perpetuatingwdeleg_getattras a stat here, since it's not directly tied to the protocol in the same way as the others are.The only difference between the two is that if another client does a GETATTR for the fh while there is already a
CB_GETATTRin flight,wdeleg_getattrgets bumped today, whereasCB_GETATTRcounter does not (since there is only a singleCB_GETATTRfor that).
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Is that difference relevant? I think Dai added
wdeleg_getattroriginally, so I'd really like to hear his opinion on this.OK, so you are actually re-litigating whether the stat should exist at all. I confess I had forgotten that context.
More or less. IIRC:
We had decided that wdeleg_getattr was sort of an oddity that didn't really belong in this /proc file. The values in that file are mostly counting other very specific protocol operations, but wdeleg_getattr is basically counting GETATTR operations from other clients while there is an outstanding write delegation on a file.
Since CB_GETATTR counts should approximate wdeleg_getattr, we decided to start counting those instead (along with all of the other callback ops). wdeleg_getattr in this series is changed to just report CB_GETATTR counts.
Quoted 9 lines
Quoted 4 lines
I’m arguing only that the netlink and procfs APIs should remain equivalent, at least initially. Deleting the stat entirely probably has no consequences right now, but let’s hear from Dai when the US west coast wakes up.
Another option is to keep the stat but report it some other way. It is arguable whether nfsstat is the proper mechanism to report the different reasons for sending a
CB_GETATTR.
Sure. We could add some way to keep reporting it (or just keep using /proc for this). AFAIK though, there is no (public) consumer of this stat. nfs-utils doesn't look at it.
To wit: I'd like to head a clear use-case for this stat before we decide to we need to count it like we did before.
Signature · 1 line
Jeff Layton <[email protected]>
Dai Ngo
reviewerRe: [PATCH v6 3/6] nfsd: implement server-stats-get netlink handler
In reply to Chuck LeverHi,
The|wdeleg_getattr| counter tracks the number of times a|GETATTR| operation conflicts with an outstanding write delegation. As such, the name|wdeleg_getattr| does not accurately describe what the counter represents.
Furthermore, this statistic probably does not belong in|nfsstats|. Since there are no known user-space consumers of this counter, it would be better to remove it altogether.
Thanks,
-Dai
Quoted 119 lines · Chuck Lever
Quoted 105 lines · Chuck Lever
Quoted 97 lines · Jeff Layton
Quoted 82 lines · Chuck Lever
Quoted 75 lines · Jeff Layton
Quoted 66 lines · Chuck Lever
Quoted 36 lines · Jeff Layton
(quoted diff) +36
-0+ /* Per-version procedure counts */ + if (statp->vs_count) { + static const int proc_attrs[] = { + [2] = NFSD_A_SERVER_STATS_PROC2_OPS, + [3] = NFSD_A_SERVER_STATS_PROC3_OPS, + [4] = NFSD_A_SERVER_STATS_PROC4_OPS, + }; + unsigned int i; + + for (i = 0; i < prog->pg_nvers && + i < ARRAY_SIZE(proc_attrs); i++) { + if (!prog->pg_vers[i] || + !statp->vs_count[i]) + continue; + if (!proc_attrs[i]) + continue; + if (nfsd_nl_fill_proc_ops(skb, + proc_attrs[i], + statp->vs_count[i], + prog->pg_vers[i]->vs_nproc)) + goto err_cancel; + } + } + +#ifdef CONFIG_NFSD_V4 + /* NFSv4 individual operation counts */ + for (int i = 0; i <= LAST_NFS4_OP; i++) { + struct nlattr *nest; + u64 cnt; + + cnt = percpu_counter_sum_positive( + &nn->counter[NFSD_STATS_NFS4_OP(i)]); + + nest = nla_nest_start(skb, + NFSD_A_SERVER_STATS_PROC4OPS_OPS); + if (!nest)Review
Quoted 10 lines
(quoted diff) +10
-0+ goto err_cancel; + if (nla_put_u32(skb, NFSD_A_SERVER_PROC_ENTRY_OP, i) || + nla_put_u64_64bit(skb, NFSD_A_SERVER_PROC_ENTRY_COUNT, + cnt, NFSD_A_SERVER_PROC_ENTRY_PAD)) { + nla_nest_cancel(skb, nest); + goto err_cancel; + } + nla_nest_end(skb, nest); + } +#endifThis loop open-codes the same nest that
nfsd_nl_fill_proc_ops()builds just above --nla_nest_start(),nla_put_u32(PROC_ENTRY_OP),nla_put_u64_64bit(PROC_ENTRY_COUNT),nla_nest_end()-- into the sameNFSD_A_SERVER_STATS_PROC4OPS_OPSattribute. Could the helper be generalized to take the per-op counter source so this is not a second copy of the same code?The per-version block above skips empty versions:
if (!prog->pg_vers[i] || !statp->vs_count[i]) continue;but this loop emits an entry for every op 0..LAST_NFS4_OP, zero-count ops included. Is that difference intentional? Skipping zero counts here would also trim the worst-case message size above.
There is also a counter that this dump does not emit.
/proc/net/rpc/nfsdprints awdeleg_getattrline after proc4ops:Quoted 8 lines
seq_printf(seq, "\nwdeleg_getattr %lld", percpu_counter_sum_positive(&nn->counter[NFSD_STATS_WDELEG_GETATTR]));incremented by
nfsd_stats_wdeleg_getattr_inc(). Since the goal is to expose the statistics currently available via/proc/net/rpc/nfsd, shouldwdeleg_getattrget an attribute here too, so nfsstat over netlink does not drop it relative to the procfs path?Quoted 6 lines
The only thing in this review that is not addressed in my current series is the above comment, and I think we had agreed in an earlier email thread that this was approximately the same value as the counter for
CB_GETATTRcalls and so it's good enough for this purpose.Are you still ok with this approach?
Quoted 9 lines
In spite of the email quoting, I’ve still forgotten most of the context of that conversation.
Looking at this now, I think someone later could get an urge to add the
wdeleg_getattrstatistic to netlink… for symmetry, because it smells like technical debt, or it might actually be useful for something.IMHO we should include it now.
IDGI: what's the point of collecting that stat separately when we have stats for
CB_GETATTRalready? I'd prefer to avoid perpetuatingwdeleg_getattras a stat here, since it's not directly tied to the protocol in the same way as the others are.Quoted 7 lines
The only difference between the two is that if another client does a GETATTR for the fh while there is already a
CB_GETATTRin flight,wdeleg_getattrgets bumped today, whereasCB_GETATTRcounter does not (since there is only a singleCB_GETATTRfor that).Is that difference relevant? I think Dai added
wdeleg_getattroriginally, so I'd really like to hear his opinion on this.Quoted 7 lines
OK, so you are actually re-litigating whether the stat should exist at all. I confess I had forgotten that context.
I’m arguing only that the netlink and procfs APIs should remain equivalent, at least initially. Deleting the stat entirely probably has no consequences right now, but let’s hear from Dai when the US west coast wakes up.
Another option is to keep the stat but report it some other way. It is arguable whether nfsstat is the proper mechanism to report the different reasons for sending a
CB_GETATTR.
Jeff Layton
maintainerPATCH v6 4/6 sunrpc: remove unused svc_version vs_count field
In reply to Jeff LaytonNow that svc_seq_show() and the nfsd netlink stats handler both use the per-netns svc_stat vs_count arrays, the global per-version vs_count percpu counters are no longer read by anything. Remove the vs_count field from struct svc_version and all the associated DEFINE_PER_CPU_ALIGNED arrays and initializers across nfsd, lockd, and the NFS client callback service.
--- fs/lockd/svc4proc.c | 4 ---- fs/lockd/svcproc.c | 7 ------- fs/nfs/callback_xdr.c | 6 ------ fs/nfsd/localio.c | 3 --- fs/nfsd/nfs2acl.c | 3 --- fs/nfsd/nfs3acl.c | 3 --- fs/nfsd/nfs3proc.c | 3 --- fs/nfsd/nfs4proc.c | 3 --- fs/nfsd/nfsproc.c | 3 --- include/linux/sunrpc/svc.h | 1 - net/sunrpc/svc.c | 3 --- 11 files changed, 39 deletions(-)
fs/lockd/svc4proc.c +0 -4
@@ -1420,14 +1420,10 @@ union nlm4svc_xdrstore {struct nlm4_shareres_wrapper shareres; }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nlm4svc_call_counters[ARRAY_SIZE(nlm4svc_procedures)]); -const struct svc_version nlmsvc_version4 = { .vs_vers =4 , .vs_nproc =ARRAY_SIZE (nlm4svc_procedures), .vs_proc = nlm4svc_procedures, - .vs_count = nlm4svc_call_counters, .vs_dispatch = nlmsvc_dispatch, .vs_xdrsize =sizeof (union nlm4svc_xdrstore ), };
fs/lockd/svcproc.c +0 -7
@@ -1433,25 +1433,18 @@ union nlmsvc_xdrstore {* NLMv1 defines only procedures 1 - 15. Linux lockd also implements * procedures 0 (NULL) and 16 (SM_NOTIFY). */ -static DEFINE_PER_CPU_ALIGNED(unsigned long, nlm1svc_call_counters[17]); -const struct svc_version nlmsvc_version1 = { .vs_vers =1 , .vs_nproc =17 , .vs_proc = nlmsvc_procedures, - .vs_count = nlm1svc_call_counters, .vs_dispatch = nlmsvc_dispatch, .vs_xdrsize =sizeof (union nlmsvc_xdrstore ), }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nlm3svc_call_counters[ARRAY_SIZE(nlmsvc_procedures)]); -const struct svc_version nlmsvc_version3 = { .vs_vers =3 , .vs_nproc =ARRAY_SIZE (nlmsvc_procedures), .vs_proc = nlmsvc_procedures, - .vs_count = nlm3svc_call_counters, .vs_dispatch = nlmsvc_dispatch, .vs_xdrsize =sizeof (union nlmsvc_xdrstore ), };
fs/nfs/callback_xdr.c +0 -6
@@ -1090,26 +1090,20 @@ static const struct svc_procedure nfs4_callback_procedures1[] = { } }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfs4_callback_count1[ARRAY_SIZE(nfs4_callback_procedures1)]);const struct svc_version nfs4_callback_version1 = { .vs_vers =1 , .vs_nproc =ARRAY_SIZE (nfs4_callback_procedures1), .vs_proc = nfs4_callback_procedures1, - .vs_count = nfs4_callback_count1, .vs_xdrsize =NFS4_CALLBACK_XDRSIZE , .vs_dispatch = nfs_callback_dispatch, .vs_hidden = true, .vs_need_cong_ctrl = true, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfs4_callback_count4[ARRAY_SIZE(nfs4_callback_procedures1)]);const struct svc_version nfs4_callback_version4 = { .vs_vers =4 , .vs_nproc =ARRAY_SIZE (nfs4_callback_procedures1), .vs_proc = nfs4_callback_procedures1, - .vs_count = nfs4_callback_count4, .vs_xdrsize =NFS4_CALLBACK_XDRSIZE , .vs_dispatch = nfs_callback_dispatch, .vs_hidden = true,
fs/nfsd/localio.c +0 -3
@@ -210,14 +210,11 @@ static const struct svc_procedure localio_procedures1[] = { };#define LOCALIO_NR_PROCEDURES ARRAY_SIZE(localio_procedures1) -static DEFINE_PER_CPU_ALIGNED(unsigned long, - localio_count[LOCALIO_NR_PROCEDURES]);const struct svc_version localio_version1 = { .vs_vers =1 , .vs_nproc =LOCALIO_NR_PROCEDURES , .vs_proc = localio_procedures1, .vs_dispatch = nfsd_dispatch, - .vs_count = localio_count, .vs_xdrsize =XDR_QUADLEN (UUID_SIZE ), .vs_hidden = true, };
fs/nfsd/nfs2acl.c +0 -3
@@ -389,13 +389,10 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { }, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfsd_acl_count2[ARRAY_SIZE(nfsd_acl_procedures2)]);const struct svc_version nfsd_acl_version2 = { .vs_vers =2 , .vs_nproc =ARRAY_SIZE (nfsd_acl_procedures2), .vs_proc = nfsd_acl_procedures2, - .vs_count = nfsd_acl_count2, .vs_dispatch = nfsd_dispatch, .vs_xdrsize =NFS3_SVC_XDRSIZE , };
fs/nfsd/nfs3acl.c +0 -3
@@ -278,13 +278,10 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = { }, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfsd_acl_count3[ARRAY_SIZE(nfsd_acl_procedures3)]);const struct svc_version nfsd_acl_version3 = { .vs_vers =3 , .vs_nproc =ARRAY_SIZE (nfsd_acl_procedures3), .vs_proc = nfsd_acl_procedures3, - .vs_count = nfsd_acl_count3, .vs_dispatch = nfsd_dispatch, .vs_xdrsize =NFS3_SVC_XDRSIZE , };
fs/nfsd/nfs3proc.c +0 -3
@@ -1108,13 +1108,10 @@ static const struct svc_procedure nfsd_procedures3[22] = { }, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfsd_count3[ARRAY_SIZE(nfsd_procedures3)]);const struct svc_version nfsd_version3 = { .vs_vers =3 , .vs_nproc =ARRAY_SIZE (nfsd_procedures3), .vs_proc = nfsd_procedures3, .vs_dispatch = nfsd_dispatch, - .vs_count = nfsd_count3, .vs_xdrsize =NFS3_SVC_XDRSIZE , };
fs/nfsd/nfs4proc.c +0 -3
@@ -4177,13 +4177,10 @@ static const struct svc_procedure nfsd_procedures4[2] = { }, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfsd_count4[ARRAY_SIZE(nfsd_procedures4)]);const struct svc_version nfsd_version4 = { .vs_vers =4 , .vs_nproc =ARRAY_SIZE (nfsd_procedures4), .vs_proc = nfsd_procedures4, - .vs_count = nfsd_count4, .vs_dispatch = nfsd_dispatch, .vs_xdrsize =NFS4_SVC_XDRSIZE , .vs_rpcb_optnl = true,
fs/nfsd/nfsproc.c +0 -3
@@ -845,13 +845,10 @@ static const struct svc_procedure nfsd_procedures2[18] = { }, }; -static DEFINE_PER_CPU_ALIGNED(unsigned long, - nfsd_count2[ARRAY_SIZE(nfsd_procedures2)]);const struct svc_version nfsd_version2 = { .vs_vers =2 , .vs_nproc =ARRAY_SIZE (nfsd_procedures2), .vs_proc = nfsd_procedures2, - .vs_count = nfsd_count2, .vs_dispatch = nfsd_dispatch, .vs_xdrsize =NFS2_SVC_XDRSIZE , };
include/linux/sunrpc/svc.h +0 -1
@@ -408,7 +408,6 @@ struct svc_version {u32 vs_vers;/* version number */ u32 vs_nproc;/* number of procedures */ const struct svc_procedure *vs_proc;/* per-procedure info */ -unsigned long __percpu *vs_count;/* call counts */ u32 vs_xdrsize;/* xdrsize needed for this version */ /* Don't register with rpcbind */
net/sunrpc/svc.c +0 -3
@@ -1435,9 +1435,6 @@ svc_generic_init_request(struct svc_rqst *rqstp,memset (rqstp->rq_argp,0 , procp->pc_argzero);memset (rqstp->rq_resp,0 , procp->pc_ressize); -/* Bump per-procedure stats counter */ -this_cpu_inc (versp->vs_count[rqstp->rq_proc]); -/* Bump per-net per-procedure stats counter */ if (rqstp->rq_server->sv_stats && rqstp->rq_server->sv_stats->program == progp &&
Signature · 1 line
2.54.0
Jeff Layton
maintainerPATCH v6 5/6 nfsd: count NFSv4 callback operations per netns
In reply to Jeff LaytonThe NFS server tracks per-operation call counts for the forward channel (proc4ops) but keeps no statistics for the NFSv4 backchannel (callback) operations it sends to clients.
Add a per-netns array of percpu counters for callback operations, indexed by RFC 8881 callback opcode (OP_CB_GETATTR..OP_CB_OFFLOAD), and bump the relevant counter in nfsd4_run_cb(), which is hit exactly once per callback that is actually queued.
CB_GETATTR is sent when a GETATTR conflicts with an outstanding write delegation, which is roughly what the dedicated wdeleg_getattr counter tracked. The two are not identical: the old counter incremented on every such conflict, whereas the CB_GETATTR counter only counts callbacks that are actually queued, so concurrent conflicts that coalesce onto an already in-flight CB_GETATTR are now counted once rather than once per conflict. Report the procfs "wdeleg_getattr" line from the CB_GETATTR counter and drop the now-redundant NFSD_STATS_WDELEG_GETATTR counter, its helper, and its increment site.
--- fs/nfsd/netns.h | 12 +++++++++++- fs/nfsd/nfs4callback.c | 22 +++++++++++++++++++++- fs/nfsd/nfs4state.c | 2 -- fs/nfsd/nfsctl.c | 14 ++++++++++++++ fs/nfsd/stats.c | 2 +- fs/nfsd/stats.h | 5 +++-- 6 files changed, 50 insertions(+), 7 deletions(-)
fs/nfsd/netns.h +11 -1
@@ -53,11 +53,16 @@ enum {NFSD_STATS_FIRST_NFS4_OP ,/* count of individual nfsv4 operations */ NFSD_STATS_LAST_NFS4_OP =NFSD_STATS_FIRST_NFS4_OP +LAST_NFS4_OP , #define NFSD_STATS_NFS4_OP(op) (NFSD_STATS_FIRST_NFS4_OP + (op)) -NFSD_STATS_WDELEG_GETATTR ,/* count of getattr conflict with wdeleg */ #endifNFSD_STATS_COUNTERS_NUM }; +/* +* Per-netns NFSv4 callback (backchannel) per-operation counters, indexed +* directly by RFC 8881 callback opcode (OP_CB_GETATTR..OP_CB_OFFLOAD). +*/ +#define NFSD_STATS_CB_OPS_NUM (OP_CB_OFFLOAD + 1) +/* * Represents a nfsd "container". With respect to nfsv4 state tracking, the * fields of interest are the *_id_hashtbls and the *_name_tree. These track
@@ -198,6 +203,11 @@ struct nfsd_net {/* Per-netns stats counters */ struct percpu_counter counter[NFSD_STATS_COUNTERS_NUM ]; +#ifdef CONFIG_NFSD_V4 +/* Per-netns NFSv4 callback (backchannel) per-operation counters */ +struct percpu_counter cb_counter[NFSD_STATS_CB_OPS_NUM ]; +#endif +/* sunrpc svc stats */ struct svc_stat nfsd_svcstats;
fs/nfsd/nfs4callback.c +21 -1
@@ -1921,12 +1921,32 @@ void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,bool nfsd4_run_cb (struct nfsd4_callback *cb) {struct nfs4_client *clp = cb->cb_clp; +struct nfsd_net *nn =net_generic (clp->net, nfsd_net_id); +const struct nfsd4_callback_ops *ops = cb->cb_ops; +u32 minorversion = clp->cl_minorversion;bool queued; +/* +* Snapshot the opcode, minorversion, and the per-net pointer before +* queuing: once nfsd4_queue_cb() has queued the work, the callback (and +* the object that embeds it) may be processed and freed concurrently, so +* neither cb nor clp can be dereferenced afterward. +*/ nfsd41_cb_inflight_begin (clp); queued =nfsd4_queue_cb (cb); -if (!queued) +if (queued) { +if (ops) +nfsd_stats_cb_op_inc (nn, ops->opcode); +/* +* Minorversion > 0 callbacks prepend a CB_SEQUENCE op (see +* encode_cb_sequence4args()); count it like the forechannel +* counts SEQUENCE, so it isn't perpetually reported as zero. +*/ +if (minorversion >0 ) +nfsd_stats_cb_op_inc (nn,OP_CB_SEQUENCE ); + }else {nfsd41_cb_inflight_end (clp); + }return queued; }
fs/nfsd/nfs4state.c +0 -2
@@ -9904,7 +9904,6 @@ __be32 nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry, struct nfs4_delegation **pdp) { - struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); struct nfsd_thread_local_info *ntli = rqstp->rq_private;struct file_lock_context *ctx;struct nfs4_delegation *dp =NULL ;
@@ -9944,7 +9943,6 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry,return 0 ; } -nfsd_stats_wdeleg_getattr_inc (nn);refcount_inc (&dp->dl_stid.sc_count); ncf = &dp->dl_cb_fattr;nfs4_cb_getattr (&dp->dl_cb_fattr);
fs/nfsd/nfsctl.c +14 -0
@@ -2691,6 +2691,13 @@ static __net_init int nfsd_net_init(struct net *net)if (retval) goto out_repcache_error; +#ifdef CONFIG_NFSD_V4 + retval =percpu_counter_init_many (nn->cb_counter,0 ,GFP_KERNEL , +NFSD_STATS_CB_OPS_NUM ); +if (retval) + goto out_cb_counter_error; +#endif +memset (&nn->nfsd_svcstats,0 ,sizeof (nn->nfsd_svcstats)); nn->nfsd_svcstats.program = &nfsd_programs[0 ]; retval =svc_stat_alloc_counts (&nn->nfsd_svcstats);
@@ -2719,6 +2726,10 @@ static __net_init int nfsd_net_init(struct net *net) out_svcstats_error:svc_stat_free_counts (&nn->nfsd_svcstats); out_proc_error: +#ifdef CONFIG_NFSD_V4 +percpu_counter_destroy_many (nn->cb_counter,NFSD_STATS_CB_OPS_NUM ); +out_cb_counter_error: +#endif percpu_counter_destroy_many (nn->counter,NFSD_STATS_COUNTERS_NUM ); out_repcache_error:nfsd_idmap_shutdown (net);
@@ -2759,6 +2770,9 @@ static __net_exit void nfsd_net_exit(struct net *net)nfsd_net_cb_shutdown (nn);nfsd_proc_stat_shutdown (net);svc_stat_free_counts (&nn->nfsd_svcstats); +#ifdef CONFIG_NFSD_V4 +percpu_counter_destroy_many (nn->cb_counter,NFSD_STATS_CB_OPS_NUM ); +#endif percpu_counter_destroy_many (nn->counter,NFSD_STATS_COUNTERS_NUM );nfsd_idmap_shutdown (net);nfsd_export_shutdown (net);
fs/nfsd/stats.c +1 -1
@@ -63,7 +63,7 @@ static int nfsd_show(struct seq_file *seq, void *v) percpu_counter_sum_positive(&nn->counter[NFSD_STATS_NFS4_OP(i)])); }seq_printf (seq,"\nwdeleg_getattr %lld" , -percpu_counter_sum_positive (&nn->counter[NFSD_STATS_WDELEG_GETATTR ])); +percpu_counter_sum_positive (&nn->cb_counter[OP_CB_GETATTR ]));seq_putc (seq,'\n' );#endif
fs/nfsd/stats.h +3 -2
@@ -68,9 +68,10 @@ static inline void nfsd_stats_drc_mem_usage_sub(struct nfsd_net *nn, s64 amount) }#ifdef CONFIG_NFSD_V4 -static inline void nfsd_stats_wdeleg_getattr_inc (struct nfsd_net *nn) +static inline void nfsd_stats_cb_op_inc (struct nfsd_net *nn,u32 opcode) { -percpu_counter_inc (&nn->counter[NFSD_STATS_WDELEG_GETATTR ]); +if (opcode >=OP_CB_GETATTR && opcode <=OP_CB_OFFLOAD ) +percpu_counter_inc (&nn->cb_counter[opcode]); }#endif #endif /* _NFSD_STATS_H */
Signature · 1 line
2.54.0
Chuck Lever
maintainerRe: [PATCH v6 5/6] nfsd: count NFSv4 callback operations per netns
Quoted 18 lines · Jeff Layton
The NFS server tracks per-operation call counts for the forward channel (proc4ops) but keeps no statistics for the NFSv4 backchannel (callback) operations it sends to clients.
Add a per-netns array of percpu counters for callback operations, indexed by RFC 8881 callback opcode (OP_CB_GETATTR..OP_CB_OFFLOAD), and bump the relevant counter in
nfsd4_run_cb(), which is hit exactly once per callback that is actually queued.fs/nfsd/nfs4callback.c +1
-0@@ -1921,12 +1921,32 @@ void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,bool nfsd4_run_cb (struct nfsd4_callback *cb) {struct nfs4_client *clp = cb->cb_clp; +struct nfsd_net *nn =net_generic (clp->net, nfsd_net_id);
fs/nfsd/nfs4callback.c void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp, · line 1934
Quoted 10 lines
fs/nfsd/nfs4callback.c +8
-0+const struct nfsd4_callback_ops *ops = cb->cb_ops; +u32 minorversion = clp->cl_minorversion;bool queued; +/* +* Snapshot the opcode, minorversion, and the per-net pointer before +* queuing: once nfsd4_queue_cb() has queued the work, the callback (and +* the object that embeds it) may be processed and freed concurrently, so +* neither cb nor clp can be dereferenced afterward. +*/
The comment says the opcode is snapshotted before queuing, but the code captures the cb_ops pointer and reads ops->opcode after nfsd4_queue_cb() returns:
if (ops) nfsd_stats_cb_op_inc(nn, ops->opcode);
That read is safe because every cb_ops points at a file-scope static const nfsd4_callback_ops (nfsd4_cb_recall_ops, nfsd4_cb_getattr_ops, and so on), which outlives cb, unlike cb and clp themselves. Would the comment read more accurately as snapshotting the cb_ops pointer rather than the opcode, perhaps noting that ops stays valid afterward because it points at static data?
fs/nfsd/nfs4callback.c void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp, · line 1946
Quoted 13 lines
fs/nfsd/nfs4callback.c +10
-1nfsd41_cb_inflight_begin (clp); queued =nfsd4_queue_cb (cb); -if (!queued) +if (queued) { +if (ops) +nfsd_stats_cb_op_inc (nn, ops->opcode); +/* +* Minorversion > 0 callbacks prepend a CB_SEQUENCE op (see +* encode_cb_sequence4args()); count it like the forechannel +* counts SEQUENCE, so it isn't perpetually reported as zero. +*/ +if (minorversion >0 ) +nfsd_stats_cb_op_inc (nn,OP_CB_SEQUENCE );
Can this inflate the OP_CB_SEQUENCE count for callbacks that never send a CB_SEQUENCE? The increment fires whenever minorversion > 0, including the ops == NULL case. The only callback with cb_ops == NULL is the null probe:
nfsd4_init_cb(&clp->cl_cb_null, clp, NULL, NFSPROC4_CLNT_CB_NULL);
which nfsd4_probe_callback() and nfsd4_shutdown_callback() queue via nfsd4_run_cb(&clp->cl_cb_null). For a 4.1+ client, nfsd4_run_cb_work() drops that work item without sending any RPC:
/*
* Don't send probe messages for 4.1 or later.
*/
if (!cb->cb_ops && clp->cl_minorversion) {
nfsd4_mark_cb_state(clp, NFSD4_CB_UP);
nfsd41_destroy_cb(cb);
return;
}So callback-channel probes, backchannel changes, and client teardown each bump OP_CB_SEQUENCE for a minorversion > 0 client even though no CB_SEQUENCE (and no callback at all) goes on the wire.
Would moving the CB_SEQUENCE increment inside the if (ops) block avoid this? A real callback on a 4.1+ client always prepends CB_SEQUENCE, and the null probe carries cb_ops == NULL, so:
if (queued) {
if (ops) {
nfsd_stats_cb_op_inc(nn, ops->opcode);
if (minorversion > 0)
nfsd_stats_cb_op_inc(nn, OP_CB_SEQUENCE);
}
} else {
nfsd41_cb_inflight_end(clp);
}Quoted 5 lines
fs/nfsd/nfs4callback.c +2
-0+ }else {nfsd41_cb_inflight_end (clp); + }return queued; }
Signature · 1 line
Chuck Lever
Jeff Layton
maintainerPATCH v6 6/6 nfsd: export NFSv4 callback op stats via netlink
In reply to Jeff LaytonAdd a proc4cb-ops nested attribute to the server-stats netlink dump, reusing the existing server-proc-entry (op/count) layout. The first dump message emits one entry per callback opcode (OP_CB_GETATTR..OP_CB_OFFLOAD) from the per-netns callback counters; the set is small and fits alongside the scalar stats, so no extra paging is needed.
This lets nfsstat report NFSv4 backchannel operation counts over netlink, including CB_GETATTR which corresponds to the procfs wdeleg_getattr line.
--- Documentation/netlink/specs/nfsd.yaml | 6 ++++++ fs/nfsd/nfsctl.c | 21 +++++++++++++++++++++ include/uapi/linux/nfsd_netlink.h | 1 + 3 files changed, 28 insertions(+)
Documentation/netlink/specs/nfsd.yaml +6 -0
@@ -410,6 +410,11 @@ attribute-sets: type: nest nested-attributes: server-proc-entry multi-attr: true + - + name: proc4cb-ops + type: nest + nested-attributes: server-proc-entry + multi-attr: true operations: list:
@@ -621,6 +626,7 @@ operations: - proc3-ops - proc4-ops - proc4ops-ops + - proc4cb-ops mcast-groups: list:
fs/nfsd/nfsctl.c +21 -0
@@ -2475,6 +2475,27 @@ int nfsd_nl_server_stats_get_dumpit(struct sk_buff *skb, }#ifdef CONFIG_NFSD_V4 +/* NFSv4 callback (backchannel) per-operation counts */ +for (int op =OP_CB_GETATTR ; op <=OP_CB_OFFLOAD ; op++) { +struct nlattr *nest; +u64 cnt; + + cnt =percpu_counter_sum_positive ( + &nn->cb_counter[op]); + + nest =nla_nest_start (skb, +NFSD_A_SERVER_STATS_PROC4CB_OPS ); +if (!nest) + goto err_cancel; +if (nla_put_u32 (skb,NFSD_A_SERVER_PROC_ENTRY_OP , op) || +nla_put_u64_64bit (skb,NFSD_A_SERVER_PROC_ENTRY_COUNT , + cnt,NFSD_A_SERVER_PROC_ENTRY_PAD )) { +nla_nest_cancel (skb, nest); + goto err_cancel; + } +nla_nest_end (skb, nest); + } +/* NFSv4 individual operation counts */ for (int i =0 ; i <=LAST_NFS4_OP ; i++) {struct nlattr *nest;
include/uapi/linux/nfsd_netlink.h +1 -0
@@ -254,6 +254,7 @@ enum {NFSD_A_SERVER_STATS_PROC3_OPS ,NFSD_A_SERVER_STATS_PROC4_OPS ,NFSD_A_SERVER_STATS_PROC4OPS_OPS , +NFSD_A_SERVER_STATS_PROC4CB_OPS , __NFSD_A_SERVER_STATS_MAX,NFSD_A_SERVER_STATS_MAX = (__NFSD_A_SERVER_STATS_MAX -1 )
Signature · 1 line
2.54.0