net_sched: introduce tracepoint trace_qdisc_enqueue()
Subsystems (3)
PATCH v2 1/1 net_sched: introduce tracepoint trace_qdisc_enqueue()
From: Qitao Xu <[email protected]>
Tracepoint trace_qdisc_enqueue() is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric to trace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate than trace_net_dev_queue(), because we already successfully enqueue the packet at that point.
Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.
--- include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++ net/core/dev.c | 9 +++++++++ 2 files changed, 35 insertions(+)
include/trace/events/qdisc.h +26 -0
@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue, __entry->txq_state, __entry->packets, __entry->skbaddr ) ); +TRACE_EVENT (qdisc_enqueue, + + TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb), + +TP_ARGS (qdisc, txq, skb), + +TP_STRUCT__entry ( + __field(struct Qdisc *, qdisc) + __field(void *, skbaddr) + __field(int, ifindex) + __field(u32, handle) +__field (u32, parent) + ), + +TP_fast_assign ( + __entry->qdisc = qdisc; + __entry->skbaddr = skb; + __entry->ifindex = txq->dev ? txq->dev->ifindex : 0; + __entry->handle = qdisc->handle; + __entry->parent = qdisc->parent; + ), + +TP_printk ("enqueue ifindex=%d qdisc handle=0x%X parent=0x%X skbaddr=%px" , + __entry->ifindex, __entry->handle, __entry->parent, __entry->skbaddr) +); + TRACE_EVENT(qdisc_reset, TP_PROTO(struct Qdisc *q),
net/core/dev.c +9 -0
@@ -131,6 +131,7 @@#include <trace/events/napi.h> #include <trace/events/net.h> #include <trace/events/skb.h> +#include <trace/events/qdisc.h> #include <linux/inetdevice.h> #include <linux/cpu_rmap.h> #include <linux/static_key.h>
@@ -3864,6 +3865,8 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,if (unlikely (!nolock_qdisc_is_empty (q))) { rc = q->enqueue (skb, q, &to_free) &NET_XMIT_MASK ; +if (rc ==NET_XMIT_SUCCESS ) +trace_qdisc_enqueue (q, txq, skb);__qdisc_run (q);qdisc_run_end (q);
@@ -3880,6 +3883,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, } rc = q->enqueue (skb, q, &to_free) &NET_XMIT_MASK ; +if (rc ==NET_XMIT_SUCCESS ) +trace_qdisc_enqueue (q, txq, skb); +qdisc_run (q); no_lock_out:
@@ -3924,6 +3930,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, rc =NET_XMIT_SUCCESS ; }else { rc = q->enqueue (skb, q, &to_free) &NET_XMIT_MASK ; +if (rc ==NET_XMIT_SUCCESS ) +trace_qdisc_enqueue (q, txq, skb); +if (qdisc_run_begin (q)) {if (unlikely (contended)) {spin_unlock (&q->busylock);
Signature · 1 line
2.27.0
Quoted 41 lines · Cong Wang
From: Qitao Xu <[email protected]>
Tracepoint
trace_qdisc_enqueue()is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric totrace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate thantrace_net_dev_queue(), because we already successfully enqueue the packet at that point.Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.
Reviewed-by Cong Wang <[email protected]>Cc Jamal Hadi Salim <[email protected]>Cc Jiri Pirko <[email protected]>Signed-off-by Qitao Xu <[email protected]>--- include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++ net/core/dev.c | 9 +++++++++ 2 files changed, 35 insertions(+)include/trace/events/qdisc.h +11
-0@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue, __entry->txq_state, __entry->packets, __entry->skbaddr ) ); +TRACE_EVENT(qdisc_enqueue, + + TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb), + + TP_ARGS(qdisc, txq, skb), + + TP_STRUCT__entry( + __field(struct Qdisc *, qdisc) + __field(void *, skbaddr) + __field(int, ifindex) + __field(u32, handle)
include/trace/events/qdisc.h TRACE_EVENT(qdisc_dequeue, · line 69
Quoted 10 lines
include/trace/events/qdisc.h +10
-0+__field (u32, parent) + ), + +TP_fast_assign ( + __entry->qdisc = qdisc; + __entry->skbaddr = skb; + __entry->ifindex = txq->dev ? txq->dev->ifindex :0 ; + __entry->handle = qdisc->handle; + __entry->parent = qdisc->parent; + ),
Hi qitao, cong
Why not support the txq, we get more info from txq.
and we should take care of the return value of q->enqueue, because we can know what happens in the qdisc queue(not necessary to work with qdisc:dequeue).
and we can use a tracepoint filter for the return value too. we should introduce a new function to instead of now codes, that may make the codes clean. Please review my patch for more info.
https://patchwork.kernel.org/project/netdevbpf/patch/[email protected]/
Quoted 51 lines
include/trace/events/qdisc.h +4
-0+TP_printk ("enqueue ifindex=%d qdisc handle=0x%X parent=0x%X skbaddr=%px" , + __entry->ifindex, __entry->handle, __entry->parent, __entry->skbaddr) +); + TRACE_EVENT(qdisc_reset, TP_PROTO(struct Qdisc *q),net/core/dev.c +9
-1@@ -131,6 +131,7 @@#include <trace/events/napi.h> #include <trace/events/net.h> #include <trace/events/skb.h> +#include <trace/events/qdisc.h> #include <linux/inetdevice.h> #include <linux/cpu_rmap.h> #include <linux/static_key.h> @@ -3864,6 +3865,8 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,if (unlikely (!nolock_qdisc_is_empty (q))) { rc = q->enqueue (skb, q, &to_free) &NET_XMIT_MASK ; +if (rc ==NET_XMIT_SUCCESS ) +trace_qdisc_enqueue (q, txq, skb);__qdisc_run (q);qdisc_run_end (q);@@ -3880,6 +3883,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, } rc = q->enqueue (skb, q, &to_free) &NET_XMIT_MASK ; +if (rc ==NET_XMIT_SUCCESS ) +trace_qdisc_enqueue (q, txq, skb); +qdisc_run (q); no_lock_out:@@ -3924,6 +3930,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, rc =NET_XMIT_SUCCESS ; }else { rc = q->enqueue (skb, q, &to_free) &NET_XMIT_MASK ; +if (rc ==NET_XMIT_SUCCESS ) +trace_qdisc_enqueue (q, txq, skb); +if (qdisc_run_begin (q)) {if (unlikely (contended)) {spin_unlock (&q->busylock); --2.27.0
Signature · 1 line
Best regards, Tonghao
Quoted 47 lines · Tonghao Zhang
Quoted 45 lines · Cong Wang
From: Qitao Xu <[email protected]>
Tracepoint
trace_qdisc_enqueue()is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric totrace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate thantrace_net_dev_queue(), because we already successfully enqueue the packet at that point.Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.
Reviewed-by Cong Wang <[email protected]>Cc Jamal Hadi Salim <[email protected]>Cc Jiri Pirko <[email protected]>Signed-off-by Qitao Xu <[email protected]>--- include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++ net/core/dev.c | 9 +++++++++ 2 files changed, 35 insertions(+)include/trace/events/qdisc.h +15
-0@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue, __entry->txq_state, __entry->packets, __entry->skbaddr ) ); +TRACE_EVENT(qdisc_enqueue, + + TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb), + + TP_ARGS(qdisc, txq, skb), + + TP_STRUCT__entry( + __field(struct Qdisc *, qdisc) + __field(void *, skbaddr) + __field(int, ifindex) + __field(u32, handle) + __field(u32, parent) + ), + + TP_fast_assign(
Quoted 8 lines
include/trace/events/qdisc.h TRACE_EVENT(qdisc_dequeue, · line 69
Quoted 6 lines
include/trace/events/qdisc.h +6
-0+ __entry->qdisc = qdisc; + __entry->skbaddr = skb; + __entry->ifindex = txq->dev ? txq->dev->ifindex :0 ; + __entry->handle = qdisc->handle; + __entry->parent = qdisc->parent; + ),Hi qitao, cong
Why not support the txq, we get more info from txq.
Because we only want to calculate queueing latency, not anything else. If you need it, you are welcome to add anything reasonable in the future, it won't break ABI (see 3dd344ea84e122f791ab).
Quoted 4 lines
and we should take care of the return value of
q->enqueue, because we can know what happens in the qdisc queue(not necessary to work with qdisc:dequeue).
and we can use a tracepoint filter for the return value too.
Disagree. Because we really have no interest in dropped packets. Even if we really do, we could trace kfree_skb(), not really here.
Quoted 2 lines
we should introduce a new function to instead of now codes, that may make the codes clean. Please review my patch for more info.
Just 3 lines of code, it is totally personal taste.
I did review it. Like I said, %p does not work. Have you tested your patches? ;)
Thanks.
Quoted 58 lines · Cong Wang
Quoted 47 lines · Tonghao Zhang
Quoted 45 lines · Cong Wang
From: Qitao Xu <[email protected]>
Tracepoint
trace_qdisc_enqueue()is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric totrace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate thantrace_net_dev_queue(), because we already successfully enqueue the packet at that point.Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.
Reviewed-by Cong Wang <[email protected]>Cc Jamal Hadi Salim <[email protected]>Cc Jiri Pirko <[email protected]>Signed-off-by Qitao Xu <[email protected]>--- include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++ net/core/dev.c | 9 +++++++++ 2 files changed, 35 insertions(+)include/trace/events/qdisc.h +15
-0@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue, __entry->txq_state, __entry->packets, __entry->skbaddr ) ); +TRACE_EVENT(qdisc_enqueue, + + TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb), + + TP_ARGS(qdisc, txq, skb), + + TP_STRUCT__entry( + __field(struct Qdisc *, qdisc) + __field(void *, skbaddr) + __field(int, ifindex) + __field(u32, handle) + __field(u32, parent) + ), + + TP_fast_assign(Quoted 8 lines
include/trace/events/qdisc.h TRACE_EVENT(qdisc_dequeue, · line 69
Quoted 6 lines
include/trace/events/qdisc.h +6
-0+ __entry->qdisc = qdisc; + __entry->skbaddr = skb; + __entry->ifindex = txq->dev ? txq->dev->ifindex :0 ; + __entry->handle = qdisc->handle; + __entry->parent = qdisc->parent; + ),Hi qitao, cong
Why not support the txq, we get more info from txq.
Quoted 3 lines
Because we only want to calculate queueing latency, not anything else. If you need it, you are welcome to add anything reasonable in the future, it won't break ABI (see
3dd344ea84e122f791ab).
Thanks!
Quoted 7 lines
Quoted 4 lines
and we should take care of the return value of
q->enqueue, because we can know what happens in the qdisc queue(not necessary to work with qdisc:dequeue).
and we can use a tracepoint filter for the return value too.Disagree. Because we really have no interest in dropped packets. Even if we really do, we could trace
kfree_skb(), not really here.
The qdisc returns not only the NET_XMIT_DROP, right ? skbprio_enqueue, sfq_enqueue and red_enqueue may return the NET_XMIT_CN.
Quoted 9 lines
Quoted 2 lines
we should introduce a new function to instead of now codes, that may make the codes clean. Please review my patch for more info.
Just 3 lines of code, it is totally personal taste.
I did review it. Like I said, %p does not work. Have you tested your patches? ;)
Yes, I tested them, I didn't find the error. my patch is based on commit id 89212e160b81e778f829b89743570665810e3b13
Quoted 1 line
Thanks.
Signature · 1 line
Best regards, Tonghao
Quoted 69 lines · Tonghao Zhang
Quoted 58 lines · Cong Wang
Quoted 47 lines · Tonghao Zhang
Quoted 45 lines · Cong Wang
From: Qitao Xu <[email protected]>
Tracepoint
trace_qdisc_enqueue()is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric totrace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate thantrace_net_dev_queue(), because we already successfully enqueue the packet at that point.Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.
Reviewed-by Cong Wang <[email protected]>Cc Jamal Hadi Salim <[email protected]>Cc Jiri Pirko <[email protected]>Signed-off-by Qitao Xu <[email protected]>--- include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++ net/core/dev.c | 9 +++++++++ 2 files changed, 35 insertions(+)include/trace/events/qdisc.h +15
-0@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue, __entry->txq_state, __entry->packets, __entry->skbaddr ) ); +TRACE_EVENT(qdisc_enqueue, + + TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb), + + TP_ARGS(qdisc, txq, skb), + + TP_STRUCT__entry( + __field(struct Qdisc *, qdisc) + __field(void *, skbaddr) + __field(int, ifindex) + __field(u32, handle) + __field(u32, parent) + ), + + TP_fast_assign(Quoted 8 lines
include/trace/events/qdisc.h TRACE_EVENT(qdisc_dequeue, · line 69
Quoted 6 lines
include/trace/events/qdisc.h +6
-0+ __entry->qdisc = qdisc; + __entry->skbaddr = skb; + __entry->ifindex = txq->dev ? txq->dev->ifindex :0 ; + __entry->handle = qdisc->handle; + __entry->parent = qdisc->parent; + ),Hi qitao, cong
Why not support the txq, we get more info from txq.Quoted 3 lines
Because we only want to calculate queueing latency, not anything else. If you need it, you are welcome to add anything reasonable in the future, it won't break ABI (see
3dd344ea84e122f791ab).Thanks!
Quoted 5 lines
Quoted 4 lines
and we should take care of the return value of
q->enqueue, because we can know what happens in the qdisc queue(not necessary to work with qdisc:dequeue).
and we can use a tracepoint filter for the return value too.
Quoted 5 lines
Quoted 2 lines
Disagree. Because we really have no interest in dropped packets. Even if we really do, we could trace
kfree_skb(), not really here.The qdisc returns not only the
NET_XMIT_DROP, right ?skbprio_enqueue,sfq_enqueueandred_enqueuemay return theNET_XMIT_CN.
Sure, in that case a different packet is dropped, once again you can trace it with kfree_skb() if you want. What's the problem?
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines
we should introduce a new function to instead of now codes, that may make the codes clean. Please review my patch for more info.
Quoted 8 lines
Quoted 6 lines
Just 3 lines of code, it is totally personal taste.
I did review it. Like I said, %p does not work. Have you tested your patches? ;)
Yes, I tested them, I didn't find the error. my patch is based on commit id
89212e160b81e778f829b89743570665810e3b13
I seriously doubt it, because we actually used %p in the beginning too and got the same address for two different packets, this is why we have to move to %px. It is 100% reproducible, so it probably means you didn't test it at all.
Thanks.
Quoted 71 lines · Cong Wang
Quoted 69 lines · Tonghao Zhang
Quoted 58 lines · Cong Wang
Quoted 47 lines · Tonghao Zhang
Quoted 45 lines · Cong Wang
From: Qitao Xu <[email protected]>
Tracepoint
trace_qdisc_enqueue()is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric totrace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate thantrace_net_dev_queue(), because we already successfully enqueue the packet at that point.Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.
Reviewed-by Cong Wang <[email protected]>Cc Jamal Hadi Salim <[email protected]>Cc Jiri Pirko <[email protected]>Signed-off-by Qitao Xu <[email protected]>--- include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++ net/core/dev.c | 9 +++++++++ 2 files changed, 35 insertions(+)include/trace/events/qdisc.h +15
-0@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue, __entry->txq_state, __entry->packets, __entry->skbaddr ) ); +TRACE_EVENT(qdisc_enqueue, + + TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb), + + TP_ARGS(qdisc, txq, skb), + + TP_STRUCT__entry( + __field(struct Qdisc *, qdisc) + __field(void *, skbaddr) + __field(int, ifindex) + __field(u32, handle) + __field(u32, parent) + ), + + TP_fast_assign(Quoted 8 lines
include/trace/events/qdisc.h TRACE_EVENT(qdisc_dequeue, · line 69
Quoted 6 lines
include/trace/events/qdisc.h +6
-0+ __entry->qdisc = qdisc; + __entry->skbaddr = skb; + __entry->ifindex = txq->dev ? txq->dev->ifindex :0 ; + __entry->handle = qdisc->handle; + __entry->parent = qdisc->parent; + ),Hi qitao, cong
Why not support the txq, we get more info from txq.Quoted 3 lines
Because we only want to calculate queueing latency, not anything else. If you need it, you are welcome to add anything reasonable in the future, it won't break ABI (see
3dd344ea84e122f791ab).Thanks!
Quoted 5 lines
Quoted 4 lines
and we should take care of the return value of
q->enqueue, because we can know what happens in the qdisc queue(not necessary to work with qdisc:dequeue).
and we can use a tracepoint filter for the return value too.
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
Disagree. Because we really have no interest in dropped packets. Even if we really do, we could trace
kfree_skb(), not really here.The qdisc returns not only the
NET_XMIT_DROP, right ?skbprio_enqueue,sfq_enqueueandred_enqueuemay return theNET_XMIT_CN.Sure, in that case a different packet is dropped, once again you can trace it with
kfree_skb()if you want. What's the problem?
It's ok, but we can make it better. Yunsheng Lin may have explained why?
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 3 lines
we should introduce a new function to instead of now codes, that may make the codes clean. Please review my patch for more info.
Just 3 lines of code, it is totally personal taste.
I did review it. Like I said, %p does not work. Have you tested your patches? ;)
Quoted 7 lines
Quoted 2 lines
Yes, I tested them, I didn't find the error. my patch is based on commit id
89212e160b81e778f829b89743570665810e3b13I seriously doubt it, because we actually used %p in the beginning too and got the same address for two different packets, this is why we have to move to %px. It is 100% reproducible, so it probably means you didn't test it at all.
I use iperf to send the packets, not found the same address really
<idle>-0 [043] ..s. 62.493634: qdisc_enqueue:
enqueue ifindex=2 qdisc handle=0x0 parent=0x2C
skbaddr=00000000a40f93fb ret=0
<idle>-0 [043] ..s. 62.494641: qdisc_enqueue:
enqueue ifindex=2 qdisc handle=0x0 parent=0x20
skbaddr=00000000c3c53e95 ret=0
<idle>-0 [014] ..s. 64.473877: qdisc_enqueue:
enqueue ifindex=2 qdisc handle=0x0 parent=0x20
skbaddr=00000000ad610424 ret=0
<idle>-0 [014] ..s. 64.473896: qdisc_enqueue:enqueue ifindex=2 qdisc handle=0x0 parent=0x20
skbaddr=00000000112a562d ret=0
Quoted 2 lines
Thanks.
Signature · 1 line
Best regards, Tonghao
Quoted 5 lines · Tonghao Zhang
Quoted 2 lines · Cong Wang
Sure, in that case a different packet is dropped, once again you can trace it with
kfree_skb()if you want. What's the problem?It's ok, but we can make it better. Yunsheng Lin may have explained why?
Why it is better to trace dropped packets both in enqueue and in kfree_skb()? I fail to see it. You are just asking for duplications. If you do not see it by yourself, it means you don't understand or need it at all. ;)
Thanks.
Quoted 9 lines · Cong Wang
Quoted 5 lines · Tonghao Zhang
Quoted 2 lines · Cong Wang
Sure, in that case a different packet is dropped, once again you can trace it with
kfree_skb()if you want. What's the problem?It's ok, but we can make it better. Yunsheng Lin may have explained why?
Why it is better to trace dropped packets both in enqueue and in
kfree_skb()?
I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and kfree_skb()
If so, we must match when the packet is dropped and what packets. if we use the return value in trace_qdisc_requeue. It
is easy to know what happened(when, where, what packets were dropped ).
Quoted 2 lines
I fail to see it. You are just asking for duplications. If you do not see it by yourself, it means you don't understand or need it at all. ;)
I added the tracepoint in centos 8 4.18 kernel version in our servers for a long time.
Quoted 1 line
Thanks.
Signature · 1 line
Best regards, Tonghao
Quoted 5 lines · Tonghao Zhang
Quoted 3 lines · Cong Wang
Quoted 1 line · Tonghao Zhang
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines · Cong Wang
Quoted 2 lines
Sure, in that case a different packet is dropped, once again you can trace it with
kfree_skb()if you want. What's the problem?It's ok, but we can make it better. Yunsheng Lin may have explained why?
Why it is better to trace dropped packets both in enqueue and in
kfree_skb()?I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and
kfree_skb()
This is wrong, packets can be dropped for other reasons too, tracing enqueue is clearly not sufficient even if you trace it unconditionally. For a quick example, codel drops packets at dequeue rather than enqueue. ;)
Quoted 3 lines
If so, we must match when the packet is dropped and what packets. if we use the return value in
trace_qdisc_requeue. It
is easy to know whathappened(when, where, what packets were dropped ).
I am afraid you have to watch the dropped packets.
Thanks.
Quoted 10 lines · Cong Wang
Quoted 8 lines · Tonghao Zhang
Quoted 6 lines · Cong Wang
Quoted 4 lines · Tonghao Zhang
Quoted 2 lines · Cong Wang
Sure, in that case a different packet is dropped, once again you can trace it with
kfree_skb()if you want. What's the problem?
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
It's ok, but we can make it better. Yunsheng Lin may have explained why?
Why it is better to trace dropped packets both in enqueue and in
kfree_skb()?I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and
kfree_skb()This is wrong, packets can be dropped for other reasons too, tracing
no matter where the packet is dropped, we should allow user to know whether dropped in the enqueue. and the
what the return value.
Quoted 11 lines
enqueue is clearly not sufficient even if you trace it unconditionally. For a quick example, codel drops packets at dequeue rather than enqueue. ;)
Quoted 3 lines
If so, we must match when the packet is dropped and what packets. if we use the return value in
trace_qdisc_requeue. It
is easy to know whathappened(when, where, what packets were dropped ).I am afraid you have to watch the dropped packets.
Thanks.
Signature · 1 line
Best regards, Tonghao
Quoted 14 lines · Tonghao Zhang
Quoted 12 lines · Cong Wang
Quoted 10 lines · Tonghao Zhang
Quoted 8 lines · Cong Wang
Quoted 5 lines · Tonghao Zhang
Quoted 2 lines · Cong Wang
Sure, in that case a different packet is dropped, once again you can trace it with
kfree_skb()if you want. What's the problem?It's ok, but we can make it better. Yunsheng Lin may have explained why?
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
Why it is better to trace dropped packets both in enqueue and in
kfree_skb()?I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and
kfree_skb()This is wrong, packets can be dropped for other reasons too, tracing
no matter where the packet is dropped, we should allow user to know whether dropped in the enqueue. and the
what the return value.
Again you can know it by kfree_skb(). And you can not avoid kfree_skb() no matter how you change enqueue. So, I don't see your argument of saving kfree_skb() makes any sense here.
Thanks.
Quoted 20 lines · Cong Wang
Quoted 18 lines · Tonghao Zhang
Quoted 16 lines · Cong Wang
Quoted 13 lines · Tonghao Zhang
Quoted 9 lines · Cong Wang
Quoted 5 lines · Tonghao Zhang
Quoted 2 lines · Cong Wang
Sure, in that case a different packet is dropped, once again you can trace it with
kfree_skb()if you want. What's the problem?It's ok, but we can make it better. Yunsheng Lin may have explained why?
Why it is better to trace dropped packets both in enqueue and in
kfree_skb()?I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and
kfree_skb()
Quoted 7 lines
Quoted 4 lines
Quoted 1 line
This is wrong, packets can be dropped for other reasons too, tracing
no matter where the packet is dropped, we should allow user to know whether dropped in the enqueue. and the
what the return value.Again you can know it by
kfree_skb(). And you can not avoidkfree_skb()no matter how you change enqueue. So, I don't see your
No, If I know what value returned for specified qdisc , I can know what happened, not necessarily kfree_skb()
Quoted 3 lines
argument of saving
kfree_skb()makes any sense here.Thanks.
Signature · 1 line
Best regards, Tonghao
Quoted 23 lines · Tonghao Zhang
Quoted 21 lines · Cong Wang
Quoted 19 lines · Tonghao Zhang
Quoted 17 lines · Cong Wang
Quoted 5 lines · Tonghao Zhang
Quoted 3 lines · Cong Wang
Quoted 1 line · Tonghao Zhang
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines · Cong Wang
Quoted 2 lines
Sure, in that case a different packet is dropped, once again you can trace it with
kfree_skb()if you want. What's the problem?It's ok, but we can make it better. Yunsheng Lin may have explained why?
Why it is better to trace dropped packets both in enqueue and in
kfree_skb()?I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and
kfree_skb()This is wrong, packets can be dropped for other reasons too, tracing
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
no matter where the packet is dropped, we should allow user to know whether dropped in the enqueue. and the
what the return value.Again you can know it by
kfree_skb(). And you can not avoidkfree_skb()no matter how you change enqueue. So, I don't see yourNo, If I know what value returned for specified qdisc , I can know what happened, not necessarily
kfree_skb()
This is wrong. You have to trace dropped packets because you need to know when to delete the key (skb address) from the hashtable you use to calculate the latency. You save the key on enqueue and remove it on both dequeue and kfree_skb, the only difference is you only need to calculate the timestamp difference for the former.
Thanks.
Quoted 31 lines · Cong Wang
Quoted 29 lines · Tonghao Zhang
Quoted 27 lines · Cong Wang
Quoted 14 lines · Tonghao Zhang
Quoted 12 lines · Cong Wang
Quoted 10 lines · Tonghao Zhang
Quoted 8 lines · Cong Wang
Quoted 5 lines · Tonghao Zhang
Quoted 2 lines · Cong Wang
Sure, in that case a different packet is dropped, once again you can trace it with
kfree_skb()if you want. What's the problem?It's ok, but we can make it better. Yunsheng Lin may have explained why?
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
Why it is better to trace dropped packets both in enqueue and in
kfree_skb()?I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and
kfree_skb()This is wrong, packets can be dropped for other reasons too, tracing
no matter where the packet is dropped, we should allow user to know whether dropped in the enqueue. and the
what the return value.Again you can know it by
kfree_skb(). And you can not avoidkfree_skb()no matter how you change enqueue. So, I don't see your
Quoted 7 lines
Quoted 2 lines
No, If I know what value returned for specified qdisc , I can know what happened, not necessarily
kfree_skb()This is wrong. You have to trace dropped packets because you need to know when to delete the key (skb address) from the hashtable you use to calculate the latency. You save the key on enqueue and remove it on both dequeue and
kfree_skb, the only difference is you
No, we can set the timestamp or cb in skb in enqueue and check them in dequeue. we may not use the hashtable.
If we use hashtable, we still can check the return value, save them to hashtable or not.
Quoted 3 lines
only need to calculate the timestamp difference for the former.
Thanks.
Signature · 1 line
Best regards, Tonghao
Quoted 36 lines · Tonghao Zhang
Quoted 34 lines · Cong Wang
Quoted 31 lines · Tonghao Zhang
Quoted 20 lines · Cong Wang
Quoted 18 lines · Tonghao Zhang
Quoted 16 lines · Cong Wang
Quoted 13 lines · Tonghao Zhang
Quoted 9 lines · Cong Wang
Quoted 5 lines · Tonghao Zhang
Quoted 2 lines · Cong Wang
Sure, in that case a different packet is dropped, once again you can trace it with
kfree_skb()if you want. What's the problem?It's ok, but we can make it better. Yunsheng Lin may have explained why?
Why it is better to trace dropped packets both in enqueue and in
kfree_skb()?I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and
kfree_skb()Quoted 7 lines
Quoted 4 lines
Quoted 1 line
This is wrong, packets can be dropped for other reasons too, tracing
no matter where the packet is dropped, we should allow user to know whether dropped in the enqueue. and the
what the return value.Again you can know it by
kfree_skb(). And you can not avoidkfree_skb()no matter how you change enqueue. So, I don't see yourNo, If I know what value returned for specified qdisc , I can know what happened, not necessarily
kfree_skb()
Quoted 6 lines
Quoted 4 lines
This is wrong. You have to trace dropped packets because you need to know when to delete the key (skb address) from the hashtable you use to calculate the latency. You save the key on enqueue and remove it on both dequeue and
kfree_skb, the only difference is youNo, we can set the timestamp or cb in skb in enqueue and check them in dequeue. we may not use the hashtable.
Are you sure this is safe?? How do you ensure we have enough space in skb->cb[]? More importantly, why do we even modify skb in tracepoint?
Quoted 2 lines
If we use hashtable, we still can check the return value, save them to hashtable or not.
How many times do I have to repeat the return value of enqueue is not sufficient even if you trace it? Can't you just look at codel_dequeue() and tell me how a drop at dequeue can be reflected to enqueue?
Thanks.
Quoted 71 lines · Cong Wang
From: Qitao Xu <[email protected]>
Tracepoint
trace_qdisc_enqueue()is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric totrace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate thantrace_net_dev_queue(), because we already successfully enqueue the packet at that point.Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.
Reviewed-by Cong Wang <[email protected]>Cc Jamal Hadi Salim <[email protected]>Cc Jiri Pirko <[email protected]>Signed-off-by Qitao Xu <[email protected]>--- include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++ net/core/dev.c | 9 +++++++++ 2 files changed, 35 insertions(+)include/trace/events/qdisc.h +26
-0@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue, __entry->txq_state, __entry->packets, __entry->skbaddr ) ); +TRACE_EVENT (qdisc_enqueue, + + TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb), + +TP_ARGS (qdisc, txq, skb), + +TP_STRUCT__entry ( + __field(struct Qdisc *, qdisc) + __field(void *, skbaddr) + __field(int, ifindex) + __field(u32, handle) +__field (u32, parent) + ), + +TP_fast_assign ( + __entry->qdisc = qdisc; + __entry->skbaddr = skb; + __entry->ifindex = txq->dev ? txq->dev->ifindex : 0; + __entry->handle = qdisc->handle; + __entry->parent = qdisc->parent; + ), + +TP_printk ("enqueue ifindex=%d qdisc handle=0x%X parent=0x%X skbaddr=%px" , + __entry->ifindex, __entry->handle, __entry->parent, __entry->skbaddr) +); + TRACE_EVENT(qdisc_reset, TP_PROTO(struct Qdisc *q),net/core/dev.c +1
-0@@ -131,6 +131,7 @@#include <trace/events/napi.h> #include <trace/events/net.h> #include <trace/events/skb.h> +#include <trace/events/qdisc.h> #include <linux/inetdevice.h> #include <linux/cpu_rmap.h> #include <linux/static_key.h>
net/core/dev.c static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, · line 3868
Quoted 4 lines
net/core/dev.c +1
-0@@ -3864,6 +3865,8 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,if (unlikely (!nolock_qdisc_is_empty (q))) { rc = q->enqueue (skb, q, &to_free) &NET_XMIT_MASK ; +if (rc ==NET_XMIT_SUCCESS )
If NET_XMIT_CN is returned, the skb seems to be enqueued too?
Also instead of checking the rc before calling the trace_*, maybe it make more sense to add the rc to the tracepoint, so that the checking is avoided, and we are able to tell the enqueuing result of a specific skb from that tracepoint too.
net/core/dev.c static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, · line 3869
Quoted 1 line
net/core/dev.c +1
-0+trace_qdisc_enqueue (q, txq, skb);
Does it make sense to wrap the about to something like:
int sch_enqueue(....)
{
rc = q->enqueue(skb, q, &to_free)..
....
trace_qdisc_enqueue(q, txq, skb);
}So that the below code can reuse that wrapper too.
Quoted 24 lines
net/core/dev.c +6
-0__qdisc_run (q);qdisc_run_end (q);@@ -3880,6 +3883,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, } rc = q->enqueue (skb, q, &to_free) &NET_XMIT_MASK ; +if (rc ==NET_XMIT_SUCCESS ) +trace_qdisc_enqueue (q, txq, skb); +qdisc_run (q); no_lock_out:@@ -3924,6 +3930,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, rc =NET_XMIT_SUCCESS ; }else { rc = q->enqueue (skb, q, &to_free) &NET_XMIT_MASK ; +if (rc ==NET_XMIT_SUCCESS ) +trace_qdisc_enqueue (q, txq, skb); +if (qdisc_run_begin (q)) {if (unlikely (contended)) {spin_unlock (&q->busylock);
Quoted 80 lines · Yunsheng Lin
Quoted 71 lines · Cong Wang
From: Qitao Xu <[email protected]>
Tracepoint
trace_qdisc_enqueue()is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric totrace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate thantrace_net_dev_queue(), because we already successfully enqueue the packet at that point.Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.
Reviewed-by Cong Wang <[email protected]>Cc Jamal Hadi Salim <[email protected]>Cc Jiri Pirko <[email protected]>Signed-off-by Qitao Xu <[email protected]>--- include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++ net/core/dev.c | 9 +++++++++ 2 files changed, 35 insertions(+)include/trace/events/qdisc.h +26
-0@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue, __entry->txq_state, __entry->packets, __entry->skbaddr ) ); +TRACE_EVENT (qdisc_enqueue, + + TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb), + +TP_ARGS (qdisc, txq, skb), + +TP_STRUCT__entry ( + __field(struct Qdisc *, qdisc) + __field(void *, skbaddr) + __field(int, ifindex) + __field(u32, handle) +__field (u32, parent) + ), + +TP_fast_assign ( + __entry->qdisc = qdisc; + __entry->skbaddr = skb; + __entry->ifindex = txq->dev ? txq->dev->ifindex : 0; + __entry->handle = qdisc->handle; + __entry->parent = qdisc->parent; + ), + +TP_printk ("enqueue ifindex=%d qdisc handle=0x%X parent=0x%X skbaddr=%px" , + __entry->ifindex, __entry->handle, __entry->parent, __entry->skbaddr) +); + TRACE_EVENT(qdisc_reset, TP_PROTO(struct Qdisc *q),net/core/dev.c +1
-0@@ -131,6 +131,7 @@#include <trace/events/napi.h> #include <trace/events/net.h> #include <trace/events/skb.h> +#include <trace/events/qdisc.h> #include <linux/inetdevice.h> #include <linux/cpu_rmap.h> #include <linux/static_key.h> net/core/dev.c static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, · line 3868
Quoted 4 lines
net/core/dev.c +1
-0@@ -3864,6 +3865,8 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,if (unlikely (!nolock_qdisc_is_empty (q))) { rc = q->enqueue (skb, q, &to_free) &NET_XMIT_MASK ; +if (rc ==NET_XMIT_SUCCESS )If
NET_XMIT_CNis returned, the skb seems to be enqueued too?
Quoted 4 lines
Also instead of checking the rc before calling the trace_*, maybe it make more sense to add the rc to the tracepoint, so that the checking is avoided, and we are able to tell the enqueuing result of a specific skb from that tracepoint too.
Yes, I will fix it.
Quoted 10 lines
net/core/dev.c static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, · line 3869
Quoted 1 line
net/core/dev.c +1
-0+trace_qdisc_enqueue (q, txq, skb);Does it make sense to wrap the about to something like:
int
sch_enqueue(....){ rc = q->enqueue(skb, q, &to_free).. .... trace_qdisc_enqueue(q, txq, skb); }
Yes, I agree, my patch uses qdisc_enqueue_skb, because __dev_xmit_skb invoke the qdisc_xxx api.
https://patchwork.kernel.org/project/netdevbpf/patch/[email protected]/
Quoted 26 lines
So that the below code can reuse that wrapper too.
Quoted 24 lines
net/core/dev.c +6
-0__qdisc_run (q);qdisc_run_end (q);@@ -3880,6 +3883,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, } rc = q->enqueue (skb, q, &to_free) &NET_XMIT_MASK ; +if (rc ==NET_XMIT_SUCCESS ) +trace_qdisc_enqueue (q, txq, skb); +qdisc_run (q); no_lock_out:@@ -3924,6 +3930,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, rc =NET_XMIT_SUCCESS ; }else { rc = q->enqueue (skb, q, &to_free) &NET_XMIT_MASK ; +if (rc ==NET_XMIT_SUCCESS ) +trace_qdisc_enqueue (q, txq, skb); +if (qdisc_run_begin (q)) {if (unlikely (contended)) {spin_unlock (&q->busylock);
Signature · 1 line
Best regards, Tonghao
Quoted 71 lines · Yunsheng Lin
Quoted 69 lines · Cong Wang
From: Qitao Xu <[email protected]>
Tracepoint
trace_qdisc_enqueue()is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric totrace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate thantrace_net_dev_queue(), because we already successfully enqueue the packet at that point.Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.
Reviewed-by Cong Wang <[email protected]>Cc Jamal Hadi Salim <[email protected]>Cc Jiri Pirko <[email protected]>Signed-off-by Qitao Xu <[email protected]>--- include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++ net/core/dev.c | 9 +++++++++ 2 files changed, 35 insertions(+)include/trace/events/qdisc.h +26
-0@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue, __entry->txq_state, __entry->packets, __entry->skbaddr ) ); +TRACE_EVENT (qdisc_enqueue, + + TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb), + +TP_ARGS (qdisc, txq, skb), + +TP_STRUCT__entry ( + __field(struct Qdisc *, qdisc) + __field(void *, skbaddr) + __field(int, ifindex) + __field(u32, handle) +__field (u32, parent) + ), + +TP_fast_assign ( + __entry->qdisc = qdisc; + __entry->skbaddr = skb; + __entry->ifindex = txq->dev ? txq->dev->ifindex : 0; + __entry->handle = qdisc->handle; + __entry->parent = qdisc->parent; + ), + +TP_printk ("enqueue ifindex=%d qdisc handle=0x%X parent=0x%X skbaddr=%px" , + __entry->ifindex, __entry->handle, __entry->parent, __entry->skbaddr) +); + TRACE_EVENT(qdisc_reset, TP_PROTO(struct Qdisc *q),net/core/dev.c +1
-0@@ -131,6 +131,7 @@#include <trace/events/napi.h> #include <trace/events/net.h> #include <trace/events/skb.h> +#include <trace/events/qdisc.h> #include <linux/inetdevice.h> #include <linux/cpu_rmap.h>
Quoted 8 lines
net/core/dev.c static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, · line 3868
Quoted 6 lines
net/core/dev.c +1
-0#include <linux/static_key.h> @@ -3864,6 +3865,8 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,if (unlikely (!nolock_qdisc_is_empty (q))) { rc = q->enqueue (skb, q, &to_free) &NET_XMIT_MASK ; +if (rc ==NET_XMIT_SUCCESS )If
NET_XMIT_CNis returned, the skb seems to be enqueued too?
Sure. See the other reply from on why dropped packets are not interesting here.
Quoted 5 lines
Also instead of checking the rc before calling the trace_*, maybe it make more sense to add the rc to the tracepoint, so that the checking is avoided, and we are able to tell the enqueuing result of a specific skb from that tracepoint too.
Totally disagree, because trace_qdisc_dequeue() is only called for successful cases too (see dequeue_skb()), it does not make sense to let them be different.
Quoted 4 lines
net/core/dev.c static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, · line 3869
Quoted 2 lines
net/core/dev.c +1
-0+trace_qdisc_enqueue (q, txq, skb);Does it make sense to wrap the about to something like:
Nope. Because ->enqueue() is called by lower layer qdisc's too, but here we only want to track root, aka, entrance of TC. I know this may be confusing, please blame trace_qdisc_dequeue() which only tracks the exit. ;)
Thanks.