Skip to contentSkip to conversation

SUNRPC: Move call to rpc_count_iostats before rpc_call_done

PATCH In review Dave Wysochanski 5 participants 18 messages May 23, 2019 – Jun 7, 2019

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/3 SUNRPC: Move call to rpc_count_iostats before rpc_call_done

For diagnostic purposes, it would be useful to have an rpc_iostats metric of RPCs completing with tk_status < 0. Unfortunately, tk_status is reset inside the rpc_call_done functions for each operation, and the call to tally the per-op metrics comes after rpc_call_done. Refactor the call to rpc_count_iostat earlier in rpc_exit_task so we can count these RPCs completing in error.

---
 net/sunrpc/sched.c | 5 +++++
 net/sunrpc/xprt.c  | 4 ----
 2 files changed, 5 insertions(+), 4 deletions(-)
net/sunrpc/sched.c +5 -0
index 28956c70100a..543caef296e4 100644
@@ -22,6 +22,7 @@
 #include <linux/sched/mm.h>
 
 #include <linux/sunrpc/clnt.h>
+#include <linux/sunrpc/metrics.h>
 
 #include "sunrpc.h"
 
@@ -770,6 +771,10 @@ rpc_reset_task_statistics(struct rpc_task *task)
 void rpc_exit_task(struct rpc_task *task)
 {
 	task->tk_action = NULL;
+	if (task->tk_ops->rpc_count_stats)
+		task->tk_ops->rpc_count_stats(task, task->tk_calldata);
+	else if (task->tk_client)
+		rpc_count_iostats(task, task->tk_client->cl_metrics);
 	if (task->tk_ops->rpc_call_done != NULL) {
 		task->tk_ops->rpc_call_done(task, task->tk_calldata);
 		if (task->tk_action != NULL) {
net/sunrpc/xprt.c +0 -4
index d7117d241460..d4477b227a96 100644
@@ -1716,10 +1716,6 @@ void xprt_release(struct rpc_task *task)
 	}
 
 	xprt = req->rq_xprt;
-	if (task->tk_ops->rpc_count_stats != NULL)
-		task->tk_ops->rpc_count_stats(task, task->tk_calldata);
-	else if (task->tk_client)
-		rpc_count_iostats(task, task->tk_client->cl_metrics);
 	xprt_request_dequeue_all(task, req);
 	spin_lock_bh(&xprt->transport_lock);
 	xprt->ops->release_xprt(xprt, task);
Signature · 1 line
2.20.1
#1 lore raw

PATCH 2/3 SUNRPC: Use proper printk specifiers for unsigned long long

Update the printk specifiers inside _print_rpc_iostats to avoid a checkpatch warning.

---
 net/sunrpc/stats.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
net/sunrpc/stats.c +1 -1
index 71166b393732..8b2d3c58ffae 100644
@@ -224,7 +224,7 @@ static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats,
 			       int op, const struct rpc_procinfo *procs)
 {
 	_print_name(seq, op, procs);
-	seq_printf(seq, "%lu %lu %lu %Lu %Lu %Lu %Lu %Lu\n",
+	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu\n",
 		   stats->om_ops,
 		   stats->om_ntrans,
 		   stats->om_timeouts,
Signature · 1 line
2.20.1
#2 lore raw

PATCH 3/3 SUNRPC: Count ops completing with tk_status < 0

In reply to Dave Wysochanski

We often see various error conditions with NFS4.x that show up with a very high operation count all completing with tk_status < 0 in a short period of time. Add a count to rpc_iostats to record on a per-op basis the ops that complete in this manner, which will enable lower overhead diagnostics.

---
 include/linux/sunrpc/metrics.h | 7 ++++++-
 net/sunrpc/stats.c             | 8 ++++++--
 2 files changed, 12 insertions(+), 3 deletions(-)
include/linux/sunrpc/metrics.h +6 -1
index 1b3751327575..0ee3f7052846 100644
@@ -30,7 +30,7 @@
 #include <linux/ktime.h>
 #include <linux/spinlock.h>
 
-#define RPC_IOSTATS_VERS	"1.0"
+#define RPC_IOSTATS_VERS	"1.1"
 
 struct rpc_iostats {
 	spinlock_t		om_lock;
@@ -66,6 +66,11 @@ struct rpc_iostats {
 	ktime_t			om_queue,	/* queued for xmit */
 				om_rtt,		/* RPC RTT */
 				om_execute;	/* RPC execution */
+	/*
+	 * The count of operations that complete with tk_status < 0.
+	 * These statuses usually indicate error conditions.
+	 */
+	unsigned long           om_error_status;
 } ____cacheline_aligned;
 
 struct rpc_task;
net/sunrpc/stats.c +6 -2
index 8b2d3c58ffae..737414247ca7 100644
@@ -176,6 +176,8 @@ void rpc_count_iostats_metrics(const struct rpc_task *task,
 
 	execute = ktime_sub(now, task->tk_start);
 	op_metrics->om_execute = ktime_add(op_metrics->om_execute, execute);
+	if (task->tk_status < 0)
+		op_metrics->om_error_status++;
 
 	spin_unlock(&op_metrics->om_lock);
 
@@ -218,13 +220,14 @@ static void _add_rpc_iostats(struct rpc_iostats *a, struct rpc_iostats *b)
 	a->om_queue = ktime_add(a->om_queue, b->om_queue);
 	a->om_rtt = ktime_add(a->om_rtt, b->om_rtt);
 	a->om_execute = ktime_add(a->om_execute, b->om_execute);
+	a->om_error_status += b->om_error_status;
 }
 
 static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats,
 			       int op, const struct rpc_procinfo *procs)
 {
 	_print_name(seq, op, procs);
-	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu\n",
+	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu %lu\n",
 		   stats->om_ops,
 		   stats->om_ntrans,
 		   stats->om_timeouts,
@@ -232,7 +235,8 @@ static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats,
 		   stats->om_bytes_recv,
 		   ktime_to_ms(stats->om_queue),
 		   ktime_to_ms(stats->om_rtt),
-		   ktime_to_ms(stats->om_execute));
+		   ktime_to_ms(stats->om_execute),
+		   stats->om_error_status);
 }
 
 void rpc_clnt_show_stats(struct seq_file *seq, struct rpc_clnt *clnt)
Signature · 1 line
2.20.1
#3 lore raw

Re: [PATCH 3/3] SUNRPC: Count ops completing with tk_status < 0

Quoted 5 lines · Dave Wysochanski

We often see various error conditions with NFS4.x that show up with a very high operation count all completing with tk_status < 0 in a short period of time. Add a count to rpc_iostats to record on a per-op basis the ops that complete in this manner, which will enable lower overhead diagnostics.

Looks like a good idea to me.

It's too bad we can't distinguish the errors. (E.g. ESTALE on a lookup call is a lot more interesting than ENOENT.) But understood that maintaining (number of possible errors) * (number of possible ops) counters is probably overkill, so just counting the number of errors seems like a good start.

--b.

Quoted 73 lines
---
 include/linux/sunrpc/metrics.h | 7 ++++++-
 net/sunrpc/stats.c             | 8 ++++++--
 2 files changed, 12 insertions(+), 3 deletions(-)
include/linux/sunrpc/metrics.h +6 -1
index 1b3751327575..0ee3f7052846 100644
@@ -30,7 +30,7 @@
 #include <linux/ktime.h>
 #include <linux/spinlock.h>
 
-#define RPC_IOSTATS_VERS	"1.0"
+#define RPC_IOSTATS_VERS	"1.1"
 
 struct rpc_iostats {
 	spinlock_t		om_lock;
@@ -66,6 +66,11 @@ struct rpc_iostats {
 	ktime_t			om_queue,	/* queued for xmit */
 				om_rtt,		/* RPC RTT */
 				om_execute;	/* RPC execution */
+	/*
+	 * The count of operations that complete with tk_status < 0.
+	 * These statuses usually indicate error conditions.
+	 */
+	unsigned long           om_error_status;
 } ____cacheline_aligned;
 
 struct rpc_task;
net/sunrpc/stats.c +6 -2
index 8b2d3c58ffae..737414247ca7 100644
@@ -176,6 +176,8 @@ void rpc_count_iostats_metrics(const struct rpc_task *task,
 
 	execute = ktime_sub(now, task->tk_start);
 	op_metrics->om_execute = ktime_add(op_metrics->om_execute, execute);
+	if (task->tk_status < 0)
+		op_metrics->om_error_status++;
 
 	spin_unlock(&op_metrics->om_lock);
 
@@ -218,13 +220,14 @@ static void _add_rpc_iostats(struct rpc_iostats *a, struct rpc_iostats *b)
 	a->om_queue = ktime_add(a->om_queue, b->om_queue);
 	a->om_rtt = ktime_add(a->om_rtt, b->om_rtt);
 	a->om_execute = ktime_add(a->om_execute, b->om_execute);
+	a->om_error_status += b->om_error_status;
 }
 
 static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats,
 			       int op, const struct rpc_procinfo *procs)
 {
 	_print_name(seq, op, procs);
-	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu\n",
+	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu %lu\n",
 		   stats->om_ops,
 		   stats->om_ntrans,
 		   stats->om_timeouts,
@@ -232,7 +235,8 @@ static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats,
 		   stats->om_bytes_recv,
 		   ktime_to_ms(stats->om_queue),
 		   ktime_to_ms(stats->om_rtt),
-		   ktime_to_ms(stats->om_execute));
+		   ktime_to_ms(stats->om_execute),
+		   stats->om_error_status);
 }
 
 void rpc_clnt_show_stats(struct seq_file *seq, struct rpc_clnt *clnt)
Signature · 1 line
2.20.1
#4 lore raw

Re: [PATCH 3/3] SUNRPC: Count ops completing with tk_status < 0

Quoted 11 lines · On May 30, 2019, at 5:38 PM, [email protected] wrote:
Quoted 5 lines · Dave Wysochanski

We often see various error conditions with NFS4.x that show up with a very high operation count all completing with tk_status < 0 in a short period of time. Add a count to rpc_iostats to record on a per-op basis the ops that complete in this manner, which will enable lower overhead diagnostics.

Quoted 7 lines

Looks like a good idea to me.

It's too bad we can't distinguish the errors. (E.g. ESTALE on a lookup call is a lot more interesting than ENOENT.) But understood that maintaining (number of possible errors) * (number of possible ops) counters is probably overkill, so just counting the number of errors seems like a good start.

We now have trace points that can do that too.

Quoted 72 lines

--b.

Quoted 70 lines

---
include/linux/sunrpc/metrics.h | 7 ++++++-
net/sunrpc/stats.c | 8 ++++++--

2 files changed, 12 insertions(+), 3 deletions(-)
include/linux/sunrpc/metrics.h +6 -1
index 1b3751327575..0ee3f7052846 100644
@@ -30,7 +30,7 @@
 #include <linux/ktime.h>
 #include <linux/spinlock.h>
 
-#define RPC_IOSTATS_VERS	"1.0"
+#define RPC_IOSTATS_VERS	"1.1"
 
 struct rpc_iostats {
 	spinlock_t		om_lock;
@@ -66,6 +66,11 @@ struct rpc_iostats {
 	ktime_t			om_queue,	/* queued for xmit */
 				om_rtt,		/* RPC RTT */
 				om_execute;	/* RPC execution */
+	/*
+	 * The count of operations that complete with tk_status < 0.
+	 * These statuses usually indicate error conditions.
+	 */
+	unsigned long           om_error_status;
 } ____cacheline_aligned;
 
 struct rpc_task;
net/sunrpc/stats.c +6 -2
index 8b2d3c58ffae..737414247ca7 100644
@@ -176,6 +176,8 @@ void rpc_count_iostats_metrics(const struct rpc_task *task,
 
 	execute = ktime_sub(now, task->tk_start);
 	op_metrics->om_execute = ktime_add(op_metrics->om_execute, execute);
+	if (task->tk_status < 0)
+		op_metrics->om_error_status++;
 
 	spin_unlock(&op_metrics->om_lock);
 
@@ -218,13 +220,14 @@ static void _add_rpc_iostats(struct rpc_iostats *a, struct rpc_iostats *b)
 	a->om_queue = ktime_add(a->om_queue, b->om_queue);
 	a->om_rtt = ktime_add(a->om_rtt, b->om_rtt);
 	a->om_execute = ktime_add(a->om_execute, b->om_execute);
+	a->om_error_status += b->om_error_status;
 }
 
 static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats,
 			       int op, const struct rpc_procinfo *procs)
 {
 	_print_name(seq, op, procs);
-	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu\n",
+	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu %lu\n",
 		   stats->om_ops,
 		   stats->om_ntrans,
 		   stats->om_timeouts,
@@ -232,7 +235,8 @@ static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats,
 		   stats->om_bytes_recv,
 		   ktime_to_ms(stats->om_queue),
 		   ktime_to_ms(stats->om_rtt),
-		   ktime_to_ms(stats->om_execute));
+		   ktime_to_ms(stats->om_execute),
+		   stats->om_error_status);
 }
 
Quoted 3 lines
Quoted 3 lines
net/sunrpc/stats.c +0 -0
 void rpc_clnt_show_stats(struct seq_file *seq, struct rpc_clnt *clnt)
Signature · 1 line
2.20.1

--
Chuck Lever

#5 lore raw

Re: [PATCH 3/3] SUNRPC: Count ops completing with tk_status < 0

Quoted 13 lines · Chuck Lever
Quoted 13 lines · On May 30, 2019, at 5:38 PM, [email protected] wrote:
Quoted 5 lines · Dave Wysochanski

We often see various error conditions with NFS4.x that show up with a very high operation count all completing with tk_status < 0 in a short period of time. Add a count to rpc_iostats to record on a per-op basis the ops that complete in this manner, which will enable lower overhead diagnostics.

Looks like a good idea to me.

Quoted 7 lines
Quoted 5 lines

It's too bad we can't distinguish the errors. (E.g. ESTALE on a lookup call is a lot more interesting than ENOENT.) But understood that maintaining (number of possible errors) * (number of possible ops) counters is probably overkill, so just counting the number of errors seems like a good start.

We now have trace points that can do that too.

You mean, that can report every error (and its value)?

I assume having these statistics in mountstats is still useful, though.

--b.

Quoted 82 lines
Quoted 74 lines

--b.

Quoted 70 lines

---
include/linux/sunrpc/metrics.h | 7 ++++++-
net/sunrpc/stats.c | 8 ++++++--

2 files changed, 12 insertions(+), 3 deletions(-)
include/linux/sunrpc/metrics.h +6 -1
index 1b3751327575..0ee3f7052846 100644
@@ -30,7 +30,7 @@
 #include <linux/ktime.h>
 #include <linux/spinlock.h>
 
-#define RPC_IOSTATS_VERS	"1.0"
+#define RPC_IOSTATS_VERS	"1.1"
 
 struct rpc_iostats {
 	spinlock_t		om_lock;
@@ -66,6 +66,11 @@ struct rpc_iostats {
 	ktime_t			om_queue,	/* queued for xmit */
 				om_rtt,		/* RPC RTT */
 				om_execute;	/* RPC execution */
+	/*
+	 * The count of operations that complete with tk_status < 0.
+	 * These statuses usually indicate error conditions.
+	 */
+	unsigned long           om_error_status;
 } ____cacheline_aligned;
 
 struct rpc_task;
net/sunrpc/stats.c +6 -2
index 8b2d3c58ffae..737414247ca7 100644
@@ -176,6 +176,8 @@ void rpc_count_iostats_metrics(const struct rpc_task *task,
 
 	execute = ktime_sub(now, task->tk_start);
 	op_metrics->om_execute = ktime_add(op_metrics->om_execute, execute);
+	if (task->tk_status < 0)
+		op_metrics->om_error_status++;
 
 	spin_unlock(&op_metrics->om_lock);
 
@@ -218,13 +220,14 @@ static void _add_rpc_iostats(struct rpc_iostats *a, struct rpc_iostats *b)
 	a->om_queue = ktime_add(a->om_queue, b->om_queue);
 	a->om_rtt = ktime_add(a->om_rtt, b->om_rtt);
 	a->om_execute = ktime_add(a->om_execute, b->om_execute);
+	a->om_error_status += b->om_error_status;
 }
 
 static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats,
 			       int op, const struct rpc_procinfo *procs)
 {
 	_print_name(seq, op, procs);
-	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu\n",
+	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu %lu\n",
 		   stats->om_ops,
 		   stats->om_ntrans,
 		   stats->om_timeouts,
@@ -232,7 +235,8 @@ static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats,
 		   stats->om_bytes_recv,
 		   ktime_to_ms(stats->om_queue),
 		   ktime_to_ms(stats->om_rtt),
-		   ktime_to_ms(stats->om_execute));
+		   ktime_to_ms(stats->om_execute),
+		   stats->om_error_status);
 }
 
Quoted 3 lines
Quoted 3 lines
net/sunrpc/stats.c +0 -0
 void rpc_clnt_show_stats(struct seq_file *seq, struct rpc_clnt *clnt)
Signature · 1 line
2.20.1

--
Chuck Lever

#6 lore raw

Re: [PATCH 3/3] SUNRPC: Count ops completing with tk_status < 0

Quoted 24 lines · Bruce Fields
Quoted 19 lines · Chuck Lever
Quoted 18 lines · On May 30, 2019, at 5:38 PM, [email protected] wrote:
Quoted 5 lines · Dave Wysochanski

We often see various error conditions with NFS4.x that show up with a very high operation count all completing with tk_status < 0 in a short period of time. Add a count to rpc_iostats to record on a per-op basis the ops that complete in this manner, which will enable lower overhead diagnostics.

Looks like a good idea to me.

It's too bad we can't distinguish the errors. (E.g. ESTALE on a lookup call is a lot more interesting than ENOENT.) But understood that maintaining (number of possible errors) * (number of possible ops) counters is probably overkill, so just counting the number of errors seems like a good start.

Quoted 3 lines
Quoted 1 line

We now have trace points that can do that too.

You mean, that can report every error (and its value)?

Yes, the nfs_xdr_status trace point reports the error by value and symbolic name.

Quoted 84 lines

I assume having these statistics in mountstats is still useful, though.

--b.

Quoted 80 lines
Quoted 74 lines

--b.

Quoted 70 lines

---
include/linux/sunrpc/metrics.h | 7 ++++++-
net/sunrpc/stats.c | 8 ++++++--

2 files changed, 12 insertions(+), 3 deletions(-)
include/linux/sunrpc/metrics.h +6 -1
index 1b3751327575..0ee3f7052846 100644
@@ -30,7 +30,7 @@
 #include <linux/ktime.h>
 #include <linux/spinlock.h>
 
-#define RPC_IOSTATS_VERS	"1.0"
+#define RPC_IOSTATS_VERS	"1.1"
 
 struct rpc_iostats {
 	spinlock_t		om_lock;
@@ -66,6 +66,11 @@ struct rpc_iostats {
 	ktime_t			om_queue,	/* queued for xmit */
 				om_rtt,		/* RPC RTT */
 				om_execute;	/* RPC execution */
+	/*
+	 * The count of operations that complete with tk_status < 0.
+	 * These statuses usually indicate error conditions.
+	 */
+	unsigned long           om_error_status;
 } ____cacheline_aligned;
 
 struct rpc_task;
net/sunrpc/stats.c +6 -2
index 8b2d3c58ffae..737414247ca7 100644
@@ -176,6 +176,8 @@ void rpc_count_iostats_metrics(const struct rpc_task *task,
 
 	execute = ktime_sub(now, task->tk_start);
 	op_metrics->om_execute = ktime_add(op_metrics->om_execute, execute);
+	if (task->tk_status < 0)
+		op_metrics->om_error_status++;
 
 	spin_unlock(&op_metrics->om_lock);
 
@@ -218,13 +220,14 @@ static void _add_rpc_iostats(struct rpc_iostats *a, struct rpc_iostats *b)
 	a->om_queue = ktime_add(a->om_queue, b->om_queue);
 	a->om_rtt = ktime_add(a->om_rtt, b->om_rtt);
 	a->om_execute = ktime_add(a->om_execute, b->om_execute);
+	a->om_error_status += b->om_error_status;
 }
 
 static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats,
 			       int op, const struct rpc_procinfo *procs)
 {
 	_print_name(seq, op, procs);
-	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu\n",
+	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu %lu\n",
 		   stats->om_ops,
 		   stats->om_ntrans,
 		   stats->om_timeouts,
@@ -232,7 +235,8 @@ static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats,
 		   stats->om_bytes_recv,
 		   ktime_to_ms(stats->om_queue),
 		   ktime_to_ms(stats->om_rtt),
-		   ktime_to_ms(stats->om_execute));
+		   ktime_to_ms(stats->om_execute),
+		   stats->om_error_status);
 }
 
Quoted 3 lines
Quoted 3 lines
net/sunrpc/stats.c +0 -0
 void rpc_clnt_show_stats(struct seq_file *seq, struct rpc_clnt *clnt)
Signature · 1 line
2.20.1

--
Chuck Lever

--
Chuck Lever

#7 lore raw

Re: [PATCH 3/3] SUNRPC: Count ops completing with tk_status < 0

Quoted 31 lines · Chuck Lever
Quoted 31 lines

On May 30, 2019, at 6:33 PM, Bruce Fields <[email protected]> wrote:

Quoted 27 lines · Chuck Lever
Quoted 26 lines · On May 30, 2019, at 5:38 PM, [email protected] wrote:
Quoted 8 lines · Dave Wysochanski

We often see various error conditions with NFS4.x that show up with
a very high operation count all completing with tk_status < 0 in a
short period of time. Add a count to rpc_iostats to record on a
per-op basis the ops that complete in this manner, which will enable lower overhead diagnostics.

Looks like a good idea to me.

It's too bad we can't distinguish the errors. (E.g. ESTALE on a lookup
call is a lot more interesting than ENOENT.) But understood that
maintaining (number of possible errors) * (number of possible ops)
counters is probably overkill, so just counting the number of errors
seems like a good start.

Quoted 7 lines
Quoted 3 lines
Quoted 1 line

We now have trace points that can do that too.

You mean, that can report every error (and its value)?

Yes, the nfs_xdr_status trace point reports the error by value and symbolic name.

The tracepoint is very useful I agree. I don't think it will show:
a) the mount
b) the opcode

Or am I mistaken and there's a way to get those with a filter or another tracepoint?

Quoted 106 lines
Quoted 100 lines

I assume having these statistics in mountstats is still useful, though.

--b.

Quoted 94 lines
Quoted 87 lines

--b.

Quoted 83 lines

---
include/linux/sunrpc/metrics.h | 7 ++++++-
net/sunrpc/stats.c | 8 ++++++--

2 files changed, 12 insertions(+), 3 deletions(-)
include/linux/sunrpc/metrics.h +1 -1
index 1b3751327575..0ee3f7052846 100644
@@ -30,7 +30,7 @@
 #include <linux/ktime.h>
 #include <linux/spinlock.h>
 
-#define RPC_IOSTATS_VERS	"1.0"
+#define RPC_IOSTATS_VERS	"1.1"
 
 struct rpc_iostats {
 	spinlock_t		om_lock;
@@ -66,6 +66,11 @@ struct rpc_iostats {
 	ktime_t			om_queue,	/* queued for

xmit */

				om_rtt,		/* RPC RTT */
				om_execute;	/* RPC

execution */

include/linux/sunrpc/metrics.h +2 -0
+	/*
+	 * The count of operations that complete with tk_status

< 0.

include/linux/sunrpc/metrics.h +3 -0
+	 * These statuses usually indicate error conditions.
+	 */
+	unsigned long           om_error_status;

} ____cacheline_aligned;

struct rpc_task;

net/sunrpc/stats.c +2 -0
index 8b2d3c58ffae..737414247ca7 100644
@@ -176,6 +176,8 @@ void rpc_count_iostats_metrics(const
 struct rpc_task *task,
 
 	execute = ktime_sub(now, task->tk_start);
 	op_metrics->om_execute = ktime_add(op_metrics-
 >om_execute, execute);
+	if (task->tk_status < 0)
+		op_metrics->om_error_status++;
 
	spin_unlock(&op_metrics->om_lock);
net/sunrpc/stats.c +1 -0
@@ -218,13 +220,14 @@ static void _add_rpc_iostats(struct
 rpc_iostats *a, struct rpc_iostats *b)
 	a->om_queue = ktime_add(a->om_queue, b->om_queue);
 	a->om_rtt = ktime_add(a->om_rtt, b->om_rtt);
 	a->om_execute = ktime_add(a->om_execute, b-
 >om_execute);
+	a->om_error_status += b->om_error_status;
 }
 

static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats,
int op, const struct
rpc_procinfo *procs)

{
	_print_name(seq, op, procs);
net/sunrpc/stats.c +0 -1
-	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu

%llu\n",

net/sunrpc/stats.c +1 -0
+	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu

%lu\n",

		   stats->om_ops,
		   stats->om_ntrans,
		   stats->om_timeouts,
net/sunrpc/stats.c +2 -1
@@ -232,7 +235,8 @@ static void _print_rpc_iostats(struct
 seq_file *seq, struct rpc_iostats *stats,
 		   stats->om_bytes_recv,
 		   ktime_to_ms(stats->om_queue),
 		   ktime_to_ms(stats->om_rtt),
-		   ktime_to_ms(stats->om_execute));
+		   ktime_to_ms(stats->om_execute),
+		   stats->om_error_status);
 }
 
Quoted 4 lines
Quoted 4 lines

void rpc_clnt_show_stats(struct seq_file *seq, struct rpc_clnt *clnt)

Signature · 1 line
2.20.1

--
Chuck Lever

--
Chuck Lever

#8 lore raw

Re: [PATCH 3/3] SUNRPC: Count ops completing with tk_status < 0

Quoted 44 lines · Dave Wysochanski
Quoted 31 lines · Chuck Lever
Quoted 31 lines

On May 30, 2019, at 6:33 PM, Bruce Fields <[email protected]> wrote:

Quoted 27 lines · Chuck Lever
Quoted 26 lines · On May 30, 2019, at 5:38 PM, [email protected] wrote:
Quoted 8 lines · Dave Wysochanski

We often see various error conditions with NFS4.x that show up with
a very high operation count all completing with tk_status < 0 in a
short period of time. Add a count to rpc_iostats to record on a
per-op basis the ops that complete in this manner, which will enable lower overhead diagnostics.

Looks like a good idea to me.

It's too bad we can't distinguish the errors. (E.g. ESTALE on a lookup
call is a lot more interesting than ENOENT.) But understood that
maintaining (number of possible errors) * (number of possible ops)
counters is probably overkill, so just counting the number of errors
seems like a good start.

Quoted 7 lines
Quoted 3 lines
Quoted 1 line

We now have trace points that can do that too.

You mean, that can report every error (and its value)?

Yes, the nfs_xdr_status trace point reports the error by value and symbolic name.

Quoted 6 lines

The tracepoint is very useful I agree. I don't think it will show:
a) the mount
b) the opcode

Or am I mistaken and there's a way to get those with a filter or another tracepoint?

The opcode can be exposed by another trace point, but the link between the two trace points is tenuous and could be improved.

I don't believe any of the NFS trace points expose the mount. My testing is largely on a single mount so my imagination stopped there.

Quoted 100 lines
Quoted 100 lines
Quoted 99 lines

I assume having these statistics in mountstats is still useful, though.

--b.

Quoted 94 lines
Quoted 87 lines

--b.

Quoted 83 lines

---
include/linux/sunrpc/metrics.h | 7 ++++++-
net/sunrpc/stats.c | 8 ++++++--

2 files changed, 12 insertions(+), 3 deletions(-)
include/linux/sunrpc/metrics.h +1 -1
index 1b3751327575..0ee3f7052846 100644
@@ -30,7 +30,7 @@
 #include <linux/ktime.h>
 #include <linux/spinlock.h>
 
-#define RPC_IOSTATS_VERS	"1.0"
+#define RPC_IOSTATS_VERS	"1.1"
 
 struct rpc_iostats {
 	spinlock_t		om_lock;
@@ -66,6 +66,11 @@ struct rpc_iostats {
 	ktime_t			om_queue,	/* queued for

xmit */

				om_rtt,		/* RPC RTT */
				om_execute;	/* RPC

execution */

include/linux/sunrpc/metrics.h +2 -0
+	/*
+	 * The count of operations that complete with tk_status

< 0.

include/linux/sunrpc/metrics.h +3 -0
+	 * These statuses usually indicate error conditions.
+	 */
+	unsigned long           om_error_status;

} ____cacheline_aligned;

struct rpc_task;

net/sunrpc/stats.c +2 -0
index 8b2d3c58ffae..737414247ca7 100644
@@ -176,6 +176,8 @@ void rpc_count_iostats_metrics(const
 struct rpc_task *task,
 
 	execute = ktime_sub(now, task->tk_start);
 	op_metrics->om_execute = ktime_add(op_metrics-
 > om_execute, execute);
+	if (task->tk_status < 0)
+		op_metrics->om_error_status++;
 
	spin_unlock(&op_metrics->om_lock);
net/sunrpc/stats.c +1 -0
@@ -218,13 +220,14 @@ static void _add_rpc_iostats(struct
 rpc_iostats *a, struct rpc_iostats *b)
 	a->om_queue = ktime_add(a->om_queue, b->om_queue);
 	a->om_rtt = ktime_add(a->om_rtt, b->om_rtt);
 	a->om_execute = ktime_add(a->om_execute, b-
 > om_execute);
+	a->om_error_status += b->om_error_status;
 }
 

static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats,
int op, const struct
rpc_procinfo *procs)

{
	_print_name(seq, op, procs);
net/sunrpc/stats.c +0 -1
-	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu

%llu\n",

net/sunrpc/stats.c +1 -0
+	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu

%lu\n",

		   stats->om_ops,
		   stats->om_ntrans,
		   stats->om_timeouts,
net/sunrpc/stats.c +2 -1
@@ -232,7 +235,8 @@ static void _print_rpc_iostats(struct
 seq_file *seq, struct rpc_iostats *stats,
 		   stats->om_bytes_recv,
 		   ktime_to_ms(stats->om_queue),
 		   ktime_to_ms(stats->om_rtt),
-		   ktime_to_ms(stats->om_execute));
+		   ktime_to_ms(stats->om_execute),
+		   stats->om_error_status);
 }
 
Quoted 4 lines
Quoted 4 lines

void rpc_clnt_show_stats(struct seq_file *seq, struct rpc_clnt *clnt)

Signature · 1 line
2.20.1

--
Chuck Lever

Quoted 2 lines
Quoted 2 lines

--
Chuck Lever

--
Chuck Lever

#9 lore raw

Re: [PATCH 3/3] SUNRPC: Count ops completing with tk_status < 0

Quoted 57 lines · Chuck Lever
Quoted 57 lines

On Jun 3, 2019, at 2:53 PM, Dave Wysochanski <[email protected]> wrote:

Quoted 40 lines · Chuck Lever
Quoted 40 lines

On May 30, 2019, at 6:33 PM, Bruce Fields <[email protected]

Quoted 1 line

wrote:

Quoted 35 lines · Chuck Lever
Quoted 34 lines · On May 30, 2019, at 5:38 PM, [email protected] wrote:
Quoted 4 lines · Dave Wysochanski

We often see various error conditions with NFS4.x that show
up with
a very high operation count all completing with tk_status

Quoted 8 lines

< 0
in a
short period of time. Add a count to rpc_iostats to record
on a
per-op basis the ops that complete in this manner, which will
enable lower overhead diagnostics.

Looks like a good idea to me.

It's too bad we can't distinguish the errors. (E.g. ESTALE on
a lookup
call is a lot more interesting than ENOENT.) But understood
that
maintaining (number of possible errors) * (number of possible
ops)
counters is probably overkill, so just counting the number of
errors
seems like a good start.

Quoted 8 lines
Quoted 3 lines
Quoted 1 line

We now have trace points that can do that too.

You mean, that can report every error (and its value)?

Yes, the nfs_xdr_status trace point reports the error by value and
symbolic name.

The tracepoint is very useful I agree. I don't think it will show:
a) the mount
b) the opcode

Quoted 7 lines
Quoted 2 lines

Or am I mistaken and there's a way to get those with a filter or another tracepoint?

The opcode can be exposed by another trace point, but the link between
the two trace points is tenuous and could be improved.

I think the number is there but it's not decoded! This was for a WRITE completing with BAD_STATEID - very nice:

kworker/u16:0-31130 [006] .... 949028.602298: nfs4_xdr_status: operation 38: nfs status -10025 (BAD_STATEID)

Quoted 4 lines

I don't believe any of the NFS trace points expose the mount. My testing
is largely on a single mount so my imagination stopped there.

Ok thanks for the confirmation.

Quoted 114 lines
Quoted 106 lines
Quoted 106 lines
Quoted 105 lines

I assume having these statistics in mountstats is still useful, though.

--b.

Quoted 99 lines
Quoted 92 lines

--b.

Quoted 88 lines

---
include/linux/sunrpc/metrics.h | 7 ++++++-
net/sunrpc/stats.c | 8 ++++++--

2 files changed, 12 insertions(+), 3 deletions(-)
include/linux/sunrpc/metrics.h +1 -1
index 1b3751327575..0ee3f7052846 100644
@@ -30,7 +30,7 @@
 #include <linux/ktime.h>
 #include <linux/spinlock.h>
 
-#define RPC_IOSTATS_VERS	"1.0"
+#define RPC_IOSTATS_VERS	"1.1"
 
 struct rpc_iostats {
 	spinlock_t		om_lock;
@@ -66,6 +66,11 @@ struct rpc_iostats {
 	ktime_t			om_queue,	/* queued

for
xmit */

				om_rtt,		/* RPC
RTT */
				om_execute;	/* RPC

execution */

include/linux/sunrpc/metrics.h +2 -0
+	/*
+	 * The count of operations that complete with tk_status

< 0.

include/linux/sunrpc/metrics.h +3 -0
+	 * These statuses usually indicate error conditions.
+	 */
+	unsigned long           om_error_status;

} ____cacheline_aligned;

struct rpc_task;

net/sunrpc/stats.c +2 -0
index 8b2d3c58ffae..737414247ca7 100644
@@ -176,6 +176,8 @@ void rpc_count_iostats_metrics(const
 struct rpc_task *task,
 
 	execute = ktime_sub(now, task->tk_start);
 	op_metrics->om_execute = ktime_add(op_metrics-
 > om_execute, execute);
 
+	if (task->tk_status < 0)
+		op_metrics->om_error_status++;
	spin_unlock(&op_metrics->om_lock);
net/sunrpc/stats.c +1 -0
@@ -218,13 +220,14 @@ static void _add_rpc_iostats(struct
 rpc_iostats *a, struct rpc_iostats *b)
 	a->om_queue = ktime_add(a->om_queue, b->om_queue);
 	a->om_rtt = ktime_add(a->om_rtt, b->om_rtt);
 	a->om_execute = ktime_add(a->om_execute, b-
 > om_execute);
 
+	a->om_error_status += b->om_error_status;
 }
 
 static void _print_rpc_iostats(struct seq_file *seq,

struct
rpc_iostats *stats,
int op, const struct
rpc_procinfo *procs)

{
	_print_name(seq, op, procs);
net/sunrpc/stats.c +0 -1
-	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu

%llu\n",

net/sunrpc/stats.c +1 -0
+	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu

%lu\n",

		   stats->om_ops,
		   stats->om_ntrans,
		   stats->om_timeouts,
net/sunrpc/stats.c +2 -1
@@ -232,7 +235,8 @@ static void _print_rpc_iostats(struct
 seq_file *seq, struct rpc_iostats *stats,
 		   stats->om_bytes_recv,
 		   ktime_to_ms(stats->om_queue),
 		   ktime_to_ms(stats->om_rtt),
-		   ktime_to_ms(stats->om_execute));
+		   ktime_to_ms(stats->om_execute),
+		   stats->om_error_status);
 }
 
Quoted 4 lines
Quoted 4 lines

void rpc_clnt_show_stats(struct seq_file *seq, struct rpc_clnt *clnt)

Signature · 1 line
2.20.1

--
Chuck Lever

Quoted 2 lines
Quoted 2 lines

--
Chuck Lever

--
Chuck Lever

#10 lore raw

Re: [PATCH 3/3] SUNRPC: Count ops completing with tk_status < 0

Quoted 70 lines · Dave Wysochanski
Quoted 57 lines · Chuck Lever
Quoted 57 lines

On Jun 3, 2019, at 2:53 PM, Dave Wysochanski <[email protected]> wrote:

Quoted 40 lines · Chuck Lever
Quoted 40 lines

On May 30, 2019, at 6:33 PM, Bruce Fields <[email protected]

Quoted 1 line

wrote:

Quoted 35 lines · Chuck Lever
Quoted 34 lines · On May 30, 2019, at 5:38 PM, [email protected] wrote:
Quoted 4 lines · Dave Wysochanski

We often see various error conditions with NFS4.x that show
up with
a very high operation count all completing with tk_status

Quoted 8 lines

< 0
in a
short period of time. Add a count to rpc_iostats to record
on a
per-op basis the ops that complete in this manner, which will
enable lower overhead diagnostics.

Looks like a good idea to me.

It's too bad we can't distinguish the errors. (E.g. ESTALE on
a lookup
call is a lot more interesting than ENOENT.) But understood
that
maintaining (number of possible errors) * (number of possible
ops)
counters is probably overkill, so just counting the number of
errors
seems like a good start.

Quoted 8 lines
Quoted 3 lines
Quoted 1 line

We now have trace points that can do that too.

You mean, that can report every error (and its value)?

Yes, the nfs_xdr_status trace point reports the error by value and
symbolic name.

The tracepoint is very useful I agree. I don't think it will show:
a) the mount
b) the opcode

Quoted 7 lines
Quoted 2 lines

Or am I mistaken and there's a way to get those with a filter or another tracepoint?

The opcode can be exposed by another trace point, but the link between
the two trace points is tenuous and could be improved.

Quoted 4 lines

I think the number is there but it's not decoded! This was for a WRITE completing with BAD_STATEID - very nice:

kworker/u16:0-31130 [006] .... 949028.602298: nfs4_xdr_status: operation 38: nfs status -10025 (BAD_STATEID)

OK, I'm smarter than I look. This shows the NFS operation, not the RPC procedure (which would be NFS4 COMPOUND), which is the right thing to do.

Quoted 6 lines
Quoted 4 lines

I don't believe any of the NFS trace points expose the mount. My testing
is largely on a single mount so my imagination stopped there.

Ok thanks for the confirmation.

Well, it's something worth thinking carefully about: if you see a slick way to make that information available, have at it.

Quoted 105 lines
Quoted 105 lines
Quoted 105 lines
Quoted 105 lines
Quoted 104 lines

I assume having these statistics in mountstats is still useful, though.

--b.

Quoted 99 lines
Quoted 92 lines

--b.

Quoted 88 lines

---
include/linux/sunrpc/metrics.h | 7 ++++++-
net/sunrpc/stats.c | 8 ++++++--

2 files changed, 12 insertions(+), 3 deletions(-)
include/linux/sunrpc/metrics.h +1 -1
index 1b3751327575..0ee3f7052846 100644
@@ -30,7 +30,7 @@
 #include <linux/ktime.h>
 #include <linux/spinlock.h>
 
-#define RPC_IOSTATS_VERS	"1.0"
+#define RPC_IOSTATS_VERS	"1.1"
 
 struct rpc_iostats {
 	spinlock_t		om_lock;
@@ -66,6 +66,11 @@ struct rpc_iostats {
 	ktime_t			om_queue,	/* queued

for
xmit */

				om_rtt,		/* RPC
RTT */
				om_execute;	/* RPC

execution */

include/linux/sunrpc/metrics.h +2 -0
+	/*
+	 * The count of operations that complete with tk_status

< 0.

include/linux/sunrpc/metrics.h +3 -0
+	 * These statuses usually indicate error conditions.
+	 */
+	unsigned long           om_error_status;

} ____cacheline_aligned;

struct rpc_task;

net/sunrpc/stats.c +2 -0
index 8b2d3c58ffae..737414247ca7 100644
@@ -176,6 +176,8 @@ void rpc_count_iostats_metrics(const
 struct rpc_task *task,
 
 	execute = ktime_sub(now, task->tk_start);
 	op_metrics->om_execute = ktime_add(op_metrics-
 > om_execute, execute);
 
+	if (task->tk_status < 0)
+		op_metrics->om_error_status++;
	spin_unlock(&op_metrics->om_lock);
net/sunrpc/stats.c +1 -0
@@ -218,13 +220,14 @@ static void _add_rpc_iostats(struct
 rpc_iostats *a, struct rpc_iostats *b)
 	a->om_queue = ktime_add(a->om_queue, b->om_queue);
 	a->om_rtt = ktime_add(a->om_rtt, b->om_rtt);
 	a->om_execute = ktime_add(a->om_execute, b-
 > om_execute);
 
+	a->om_error_status += b->om_error_status;
 }
 
 static void _print_rpc_iostats(struct seq_file *seq,

struct
rpc_iostats *stats,
int op, const struct
rpc_procinfo *procs)

{
	_print_name(seq, op, procs);
net/sunrpc/stats.c +0 -1
-	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu

%llu\n",

net/sunrpc/stats.c +1 -0
+	seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu

%lu\n",

		   stats->om_ops,
		   stats->om_ntrans,
		   stats->om_timeouts,
net/sunrpc/stats.c +2 -1
@@ -232,7 +235,8 @@ static void _print_rpc_iostats(struct
 seq_file *seq, struct rpc_iostats *stats,
 		   stats->om_bytes_recv,
 		   ktime_to_ms(stats->om_queue),
 		   ktime_to_ms(stats->om_rtt),
-		   ktime_to_ms(stats->om_execute));
+		   ktime_to_ms(stats->om_execute),
+		   stats->om_error_status);
 }
 
Quoted 4 lines
Quoted 4 lines

void rpc_clnt_show_stats(struct seq_file *seq, struct rpc_clnt *clnt)

Signature · 1 line
2.20.1

--
Chuck Lever

Quoted 5 lines
Quoted 5 lines
Quoted 2 lines
Quoted 2 lines

--
Chuck Lever

--
Chuck Lever

--
Chuck Lever

#11 lore raw

Re: [PATCH 3/3] SUNRPC: Count ops completing with tk_status < 0

In reply to Chuck Lever
Quoted 16 lines · Chuck Lever
Quoted 16 lines · Dave Wysochanski
Quoted 9 lines · Chuck Lever
Quoted 6 lines

On May 30, 2019, at 6:33 PM, Bruce Fields <[email protected]> wrote:

Quoted 1 line · Chuck Lever

We now have trace points that can do that too.

You mean, that can report every error (and its value)?

Yes, the nfs_xdr_status trace point reports the error by value and symbolic name.

The tracepoint is very useful I agree. I don't think it will show:
a) the mount
b) the opcode

Quoted 8 lines
Quoted 2 lines

Or am I mistaken and there's a way to get those with a filter or another tracepoint?

The opcode can be exposed by another trace point, but the link between the two trace points is tenuous and could be improved.

I don't believe any of the NFS trace points expose the mount. My testing is largely on a single mount so my imagination stopped there.

Dumb question: is it possible to add more fields to tracepoints without breaking some kind of backwards compatibility?

I wonder if adding, say, an xid and an xprt pointer to tracepoints when available would help with this kind of thing.

In any case, I think Dave's stats will still be handy if only because they're on all the time.

--b.

#12 lore raw

Re: [PATCH 3/3] SUNRPC: Count ops completing with tk_status < 0

Quoted 28 lines · Bruce Fields
Quoted 27 lines · Chuck Lever
Quoted 15 lines

On Jun 3, 2019, at 2:53 PM, Dave Wysochanski <[email protected]

Quoted 1 line

wrote:

Quoted 11 lines · Chuck Lever
Quoted 7 lines

On May 30, 2019, at 6:33 PM, Bruce Fields <
[email protected]>
wrote:

Quoted 1 line · Chuck Lever

We now have trace points that can do that too.

You mean, that can report every error (and its value)?

Yes, the nfs_xdr_status trace point reports the error by value and
symbolic name.

Quoted 7 lines

The tracepoint is very useful I agree. I don't think it will show:
a) the mount
b) the opcode

Or am I mistaken and there's a way to get those with a filter or another tracepoint?

The opcode can be exposed by another trace point, but the link between
the two trace points is tenuous and could be improved.

Quoted 7 lines
Quoted 3 lines

I don't believe any of the NFS trace points expose the mount. My testing
is largely on a single mount so my imagination stopped there.

Dumb question: is it possible to add more fields to tracepoints without
breaking some kind of backwards compatibility?

Tracepoints are not to be considered an API.

Quoted 6 lines

I wonder if adding, say, an xid and an xprt pointer to tracepoints when
available would help with this kind of thing.

In any case, I think Dave's stats will still be handy if only because they're on all the time.

Given that someone who shall remain anonymous added the struct rpc_rqst to the struct xdr_stream for "debugging purposes", it should be quite possible to add both the XID and the xprt in this case.

Signature · 3 lines
Trond Myklebust
Linux NFS client maintainer, Hammerspace
[email protected]
#13 lore raw

Re: [PATCH 3/3] SUNRPC: Count ops completing with tk_status < 0

In reply to Bruce Fields
Quoted 36 lines · Bruce Fields
Quoted 23 lines · Chuck Lever
Quoted 15 lines

On Jun 3, 2019, at 2:53 PM, Dave Wysochanski <[email protected]

Quoted 1 line

wrote:

Quoted 11 lines · Chuck Lever
Quoted 7 lines

On May 30, 2019, at 6:33 PM, Bruce Fields <
[email protected]>
wrote:

Quoted 1 line · Chuck Lever

We now have trace points that can do that too.

You mean, that can report every error (and its value)?

Yes, the nfs_xdr_status trace point reports the error by value and
symbolic name.

Quoted 7 lines

The tracepoint is very useful I agree. I don't think it will show:
a) the mount
b) the opcode

Or am I mistaken and there's a way to get those with a filter or another tracepoint?

Quoted 7 lines

The opcode can be exposed by another trace point, but the link between
the two trace points is tenuous and could be improved.

I don't believe any of the NFS trace points expose the mount. My testing
is largely on a single mount so my imagination stopped there.

Dumb question: is it possible to add more fields to tracepoints without
breaking some kind of backwards compatibility?

Quoted 8 lines

I wonder if adding, say, an xid and an xprt pointer to tracepoints when
available would help with this kind of thing.

In any case, I think Dave's stats will still be handy if only because they're on all the time.

--b.

Trond or Anna, will you take this series for mountstats or are you opposed to it?

I think it is useful in conjuction with the tracepoints because it is always on and easy to know which mount is involved (we often start with a customer saying mount XYZ has some issue or is hanging). If you see problems or want other testing please let me know.

Thanks!

#14 lore raw

Re: [PATCH 3/3] SUNRPC: Count ops completing with tk_status < 0

In reply to J. Bruce Fields
Quoted 10 lines · J. Bruce Fields
Quoted 5 lines · Dave Wysochanski

We often see various error conditions with NFS4.x that show up with a very high operation count all completing with tk_status < 0 in a short period of time. Add a count to rpc_iostats to record on a per-op basis the ops that complete in this manner, which will enable lower overhead diagnostics.

Looks like a good idea to me.

Quoted 8 lines

It's too bad we can't distinguish the errors. (E.g. ESTALE on a lookup call is a lot more interesting than ENOENT.) But understood that maintaining (number of possible errors) * (number of possible ops) counters is probably overkill, so just counting the number of errors seems like a good start.

--b.

I did consider a more elaborate approach, where each code would be counted. Most likely that would need to go into debugfs. I didn't pursue that but may look into it in the future along with other work.

I do think the < 0 status counts have value even if we do not know the specific code, they are very low overhead, and always there if added to mountstats. I was envisioning using this along with something like a periodic capture of mountstats possibly using PCP and then using some statistics as a "NFS health" measure for a specific mount point. There are already some "NFS health" metrics that may be calculated with existing mountstats (for example if you're clever you can spot certain NFS4 protocol loops), but the error counts would make it much easier and reliable.

Quoted 73 lines
Quoted 73 lines
---
 include/linux/sunrpc/metrics.h | 7 ++++++-
 net/sunrpc/stats.c             | 8 ++++++--
 2 files changed, 12 insertions(+), 3 deletions(-)
include/linux/sunrpc/metrics.h +6 -1
index 1b3751327575..0ee3f7052846 100644
@@ -30,7 +30,7 @@
 #include <linux/ktime.h>
 #include <linux/spinlock.h>
 
-#define RPC_IOSTATS_VERS     "1.0"
+#define RPC_IOSTATS_VERS     "1.1"
 
 struct rpc_iostats {
      spinlock_t              om_lock;
@@ -66,6 +66,11 @@ struct rpc_iostats {
      ktime_t                 om_queue,       /* queued for xmit */
                              om_rtt,         /* RPC RTT */
                              om_execute;     /* RPC execution */
+     /*
+      * The count of operations that complete with tk_status < 0.
+      * These statuses usually indicate error conditions.
+      */
+     unsigned long           om_error_status;
 } ____cacheline_aligned;
 
 struct rpc_task;
net/sunrpc/stats.c +6 -3
index 8b2d3c58ffae..737414247ca7 100644
@@ -176,6 +176,8 @@ void rpc_count_iostats_metrics(const struct rpc_task *task,
 
      execute = ktime_sub(now, task->tk_start);
      op_metrics->om_execute = ktime_add(op_metrics->om_execute, execute);
+     if (task->tk_status < 0)
+             op_metrics->om_error_status++;
 
      spin_unlock(&op_metrics->om_lock);
 
@@ -218,13 +220,14 @@ static void _add_rpc_iostats(struct rpc_iostats *a, struct rpc_iostats *b)
      a->om_queue = ktime_add(a->om_queue, b->om_queue);
      a->om_rtt = ktime_add(a->om_rtt, b->om_rtt);
      a->om_execute = ktime_add(a->om_execute, b->om_execute);
+     a->om_error_status += b->om_error_status;
 }
 
 static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats,
                             int op, const struct rpc_procinfo *procs)
 {
      _print_name(seq, op, procs);
-     seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu\n",
+     seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %llu %lu\n",
                 stats->om_ops,
                 stats->om_ntrans,
                 stats->om_timeouts,
@@ -232,7 +235,8 @@ static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats,
                 stats->om_bytes_recv,
                 ktime_to_ms(stats->om_queue),
                 ktime_to_ms(stats->om_rtt),
-                ktime_to_ms(stats->om_execute));
+                ktime_to_ms(stats->om_execute),
+                stats->om_error_status);
 }
 
 void rpc_clnt_show_stats(struct seq_file *seq, struct rpc_clnt *clnt)
--

2.20.1

Signature · 3 lines
Dave Wysochanski
Principal Software Maintenance Engineer
T: 919-754-4024
#15 lore raw

PATCH 1/1 mountstats: add per-op error counts for mountstats command

In reply to Dave Wysochanski

Display the count of ops completing with error status (status < 0) on kernels that support it.

---
 tools/mountstats/mountstats.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
tools/mountstats/mountstats.py +3 -1
index c7fb8bb1..b8c720d5 100755
@@ -475,7 +475,9 @@ class DeviceData:
                 retrans = stats[2] - count
                 if retrans != 0:
                     print('\t%d retrans (%d%%)' % (retrans, ((retrans * 100) / count)), end=' ')
-                    print('\t%d major timeouts' % stats[3])
+                    print('\t%d major timeouts' % stats[3], end='')
+                if len(stats) >= 10 and stats[9] != 0:
+                    print('%d errors (%d%%)' % (stats[9], ((stats[9] * 100) / count)))
                 else:
                     print('')
                 print('\tavg bytes sent per op: %d\tavg bytes received per op: %d' % \
Signature · 1 line
2.20.1
#16 lore raw

PATCH v2 1/1 mountstats: add per-op error counts for mountstats command

Display the count of ops completing with error status (status < 0) on kernels that support it.

---
 tools/mountstats/mountstats.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
tools/mountstats/mountstats.py +3 -1
index c7fb8bb1..2f525f4b 100755
@@ -475,7 +475,9 @@ class DeviceData:
                 retrans = stats[2] - count
                 if retrans != 0:
                     print('\t%d retrans (%d%%)' % (retrans, ((retrans * 100) / count)), end=' ')
-                    print('\t%d major timeouts' % stats[3])
+                    print('\t%d major timeouts' % stats[3], end='')
+                if len(stats) >= 10 and stats[9] != 0:
+                    print('\t%d errors (%d%%)' % (stats[9], ((stats[9] * 100) / count)))
                 else:
                     print('')
                 print('\tavg bytes sent per op: %d\tavg bytes received per op: %d' % \
Signature · 1 line
2.20.1
#17 lore raw

Re: [PATCH v2] mountstats: add per-op error counts for mountstats command

Quoted 7 lines · Dave Wysochanski

Display the count of ops completing with error status (status < 0) on kernels that support it.

---
 tools/mountstats/mountstats.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

Committed...

steved.

Quoted 17 lines
tools/mountstats/mountstats.py +3 -1
index c7fb8bb1..2f525f4b 100755
@@ -475,7 +475,9 @@ class DeviceData:
                 retrans = stats[2] - count
                 if retrans != 0:
                     print('\t%d retrans (%d%%)' % (retrans, ((retrans * 100) / count)), end=' ')
-                    print('\t%d major timeouts' % stats[3])
+                    print('\t%d major timeouts' % stats[3], end='')
+                if len(stats) >= 10 and stats[9] != 0:
+                    print('\t%d errors (%d%%)' % (stats[9], ((stats[9] * 100) / count)))
                 else:
                     print('')
                 print('\tavg bytes sent per op: %d\tavg bytes received per op: %d' % \