Skip to contentSkip to conversation

net_sched: introduce tracepoint trace_qdisc_enqueue()

PATCH Superseded Cong Wang 3 participants 18 messages Jul 11, 2021 – Jul 15, 2021

Subsystems (3)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Cong Wang

author
#0 lore raw

PATCH v2 1/1 net_sched: introduce tracepoint trace_qdisc_enqueue()

From: Qitao Xu <[email protected]>

Tracepoint trace_qdisc_enqueue() is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric to trace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate than trace_net_dev_queue(), because we already successfully enqueue the packet at that point.

Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.

Cc Jamal Hadi Salim <[email protected]>
Cc Jiri Pirko <[email protected]>
---
 include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++
 net/core/dev.c               |  9 +++++++++
 2 files changed, 35 insertions(+)
include/trace/events/qdisc.h +26 -0
index 58209557cb3a..c3006c6b4a87 100644
@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue,
 		  __entry->txq_state, __entry->packets, __entry->skbaddr )
 );
 
+TRACE_EVENT(qdisc_enqueue,
+
+	TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb),
+
+	TP_ARGS(qdisc, txq, skb),
+
+	TP_STRUCT__entry(
+		__field(struct Qdisc *, qdisc)
+		__field(void *,	skbaddr)
+		__field(int, ifindex)
+		__field(u32, handle)
+		__field(u32, parent)
+	),
+
+	TP_fast_assign(
+		__entry->qdisc = qdisc;
+		__entry->skbaddr = skb;
+		__entry->ifindex = txq->dev ? txq->dev->ifindex : 0;
+		__entry->handle	 = qdisc->handle;
+		__entry->parent	 = qdisc->parent;
+	),
+
+	TP_printk("enqueue ifindex=%d qdisc handle=0x%X parent=0x%X skbaddr=%px",
+		  __entry->ifindex, __entry->handle, __entry->parent, __entry->skbaddr)
+);
+
 TRACE_EVENT(qdisc_reset,
 
 	TP_PROTO(struct Qdisc *q),
net/core/dev.c +9 -0
index c253c2aafe97..20b9376de301 100644
@@ -131,6 +131,7 @@
 #include <trace/events/napi.h>
 #include <trace/events/net.h>
 #include <trace/events/skb.h>
+#include <trace/events/qdisc.h>
 #include <linux/inetdevice.h>
 #include <linux/cpu_rmap.h>
 #include <linux/static_key.h>
@@ -3864,6 +3865,8 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
 			if (unlikely(!nolock_qdisc_is_empty(q))) {
 				rc = q->enqueue(skb, q, &to_free) &
 					NET_XMIT_MASK;
+				if (rc == NET_XMIT_SUCCESS)
+					trace_qdisc_enqueue(q, txq, skb);
 				__qdisc_run(q);
 				qdisc_run_end(q);
 
@@ -3880,6 +3883,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
 		}
 
 		rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
+		if (rc == NET_XMIT_SUCCESS)
+			trace_qdisc_enqueue(q, txq, skb);
+
 		qdisc_run(q);
 
 no_lock_out:
@@ -3924,6 +3930,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
 		rc = NET_XMIT_SUCCESS;
 	} else {
 		rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
+		if (rc == NET_XMIT_SUCCESS)
+			trace_qdisc_enqueue(q, txq, skb);
+
 		if (qdisc_run_begin(q)) {
 			if (unlikely(contended)) {
 				spin_unlock(&q->busylock);
Signature · 1 line
2.27.0
#1 lore raw
Quoted 41 lines · Cong Wang

From: Qitao Xu <[email protected]>

Tracepoint trace_qdisc_enqueue() is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric to trace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate than trace_net_dev_queue(), because we already successfully enqueue the packet at that point.

Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.

Cc Jamal Hadi Salim <[email protected]>
Cc Jiri Pirko <[email protected]>
---
 include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++
 net/core/dev.c               |  9 +++++++++
 2 files changed, 35 insertions(+)
include/trace/events/qdisc.h +11 -0
index 58209557cb3a..c3006c6b4a87 100644
@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue,
                  __entry->txq_state, __entry->packets, __entry->skbaddr )
 );
 
+TRACE_EVENT(qdisc_enqueue,
+
+       TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb),
+
+       TP_ARGS(qdisc, txq, skb),
+
+       TP_STRUCT__entry(
+               __field(struct Qdisc *, qdisc)
+               __field(void *, skbaddr)
+               __field(int, ifindex)
+               __field(u32, handle)

include/trace/events/qdisc.h TRACE_EVENT(qdisc_dequeue, · line 69

Quoted 10 lines
include/trace/events/qdisc.h +10 -0
+               __field(u32, parent)
+       ),
+
+       TP_fast_assign(
+               __entry->qdisc = qdisc;
+               __entry->skbaddr = skb;
+               __entry->ifindex = txq->dev ? txq->dev->ifindex : 0;
+               __entry->handle  = qdisc->handle;
+               __entry->parent  = qdisc->parent;
+       ),

Hi qitao, cong
Why not support the txq, we get more info from txq.
and we should take care of the return value of q->enqueue, because we can know what happens in the qdisc queue(not necessary to work with qdisc:dequeue).
and we can use a tracepoint filter for the return value too. we should introduce a new function to instead of now codes, that may make the codes clean. Please review my patch for more info.
https://patchwork.kernel.org/project/netdevbpf/patch/[email protected]/

Quoted 51 lines
include/trace/events/qdisc.h +4 -0
+       TP_printk("enqueue ifindex=%d qdisc handle=0x%X parent=0x%X skbaddr=%px",
+                 __entry->ifindex, __entry->handle, __entry->parent, __entry->skbaddr)
+);
+
 TRACE_EVENT(qdisc_reset,
 
        TP_PROTO(struct Qdisc *q),
net/core/dev.c +9 -1
index c253c2aafe97..20b9376de301 100644
@@ -131,6 +131,7 @@
 #include <trace/events/napi.h>
 #include <trace/events/net.h>
 #include <trace/events/skb.h>
+#include <trace/events/qdisc.h>
 #include <linux/inetdevice.h>
 #include <linux/cpu_rmap.h>
 #include <linux/static_key.h>
@@ -3864,6 +3865,8 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
                        if (unlikely(!nolock_qdisc_is_empty(q))) {
                                rc = q->enqueue(skb, q, &to_free) &
                                        NET_XMIT_MASK;
+                               if (rc == NET_XMIT_SUCCESS)
+                                       trace_qdisc_enqueue(q, txq, skb);
                                __qdisc_run(q);
                                qdisc_run_end(q);
 
@@ -3880,6 +3883,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
                }
 
                rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
+               if (rc == NET_XMIT_SUCCESS)
+                       trace_qdisc_enqueue(q, txq, skb);
+
                qdisc_run(q);
 
 no_lock_out:
@@ -3924,6 +3930,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
                rc = NET_XMIT_SUCCESS;
        } else {
                rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
+               if (rc == NET_XMIT_SUCCESS)
+                       trace_qdisc_enqueue(q, txq, skb);
+
                if (qdisc_run_begin(q)) {
                        if (unlikely(contended)) {
                                spin_unlock(&q->busylock);
--

2.27.0

Signature · 1 line
Best regards, Tonghao
author
#2 lore raw
Quoted 47 lines · Tonghao Zhang
Quoted 45 lines · Cong Wang

From: Qitao Xu <[email protected]>

Tracepoint trace_qdisc_enqueue() is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric to trace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate than trace_net_dev_queue(), because we already successfully enqueue the packet at that point.

Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.

Cc Jamal Hadi Salim <[email protected]>
Cc Jiri Pirko <[email protected]>
---
 include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++
 net/core/dev.c               |  9 +++++++++
 2 files changed, 35 insertions(+)
include/trace/events/qdisc.h +15 -0
index 58209557cb3a..c3006c6b4a87 100644
@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue,
                  __entry->txq_state, __entry->packets, __entry->skbaddr )
 );
 
+TRACE_EVENT(qdisc_enqueue,
+
+       TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb),
+
+       TP_ARGS(qdisc, txq, skb),
+
+       TP_STRUCT__entry(
+               __field(struct Qdisc *, qdisc)
+               __field(void *, skbaddr)
+               __field(int, ifindex)
+               __field(u32, handle)
+               __field(u32, parent)
+       ),
+
+       TP_fast_assign(
Quoted 8 lines

include/trace/events/qdisc.h TRACE_EVENT(qdisc_dequeue, · line 69

Quoted 6 lines
include/trace/events/qdisc.h +6 -0
+               __entry->qdisc = qdisc;
+               __entry->skbaddr = skb;
+               __entry->ifindex = txq->dev ? txq->dev->ifindex : 0;
+               __entry->handle  = qdisc->handle;
+               __entry->parent  = qdisc->parent;
+       ),

Hi qitao, cong
Why not support the txq, we get more info from txq.

Because we only want to calculate queueing latency, not anything else. If you need it, you are welcome to add anything reasonable in the future, it won't break ABI (see 3dd344ea84e122f791ab).

Quoted 4 lines

and we should take care of the return value of q->enqueue, because we can know what happens in the qdisc queue(not necessary to work with qdisc:dequeue).
and we can use a tracepoint filter for the return value too.

Disagree. Because we really have no interest in dropped packets. Even if we really do, we could trace kfree_skb(), not really here.

Quoted 2 lines

we should introduce a new function to instead of now codes, that may make the codes clean. Please review my patch for more info.

Just 3 lines of code, it is totally personal taste.

Quoted 1 line

https://patchwork.kernel.org/project/netdevbpf/patch/[email protected]/

I did review it. Like I said, %p does not work. Have you tested your patches? ;)

Thanks.

#3 lore raw
Quoted 58 lines · Cong Wang
Quoted 47 lines · Tonghao Zhang
Quoted 45 lines · Cong Wang

From: Qitao Xu <[email protected]>

Tracepoint trace_qdisc_enqueue() is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric to trace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate than trace_net_dev_queue(), because we already successfully enqueue the packet at that point.

Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.

Cc Jamal Hadi Salim <[email protected]>
Cc Jiri Pirko <[email protected]>
---
 include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++
 net/core/dev.c               |  9 +++++++++
 2 files changed, 35 insertions(+)
include/trace/events/qdisc.h +15 -0
index 58209557cb3a..c3006c6b4a87 100644
@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue,
                  __entry->txq_state, __entry->packets, __entry->skbaddr )
 );
 
+TRACE_EVENT(qdisc_enqueue,
+
+       TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb),
+
+       TP_ARGS(qdisc, txq, skb),
+
+       TP_STRUCT__entry(
+               __field(struct Qdisc *, qdisc)
+               __field(void *, skbaddr)
+               __field(int, ifindex)
+               __field(u32, handle)
+               __field(u32, parent)
+       ),
+
+       TP_fast_assign(
Quoted 8 lines

include/trace/events/qdisc.h TRACE_EVENT(qdisc_dequeue, · line 69

Quoted 6 lines
include/trace/events/qdisc.h +6 -0
+               __entry->qdisc = qdisc;
+               __entry->skbaddr = skb;
+               __entry->ifindex = txq->dev ? txq->dev->ifindex : 0;
+               __entry->handle  = qdisc->handle;
+               __entry->parent  = qdisc->parent;
+       ),

Hi qitao, cong
Why not support the txq, we get more info from txq.

Quoted 3 lines

Because we only want to calculate queueing latency, not anything else. If you need it, you are welcome to add anything reasonable in the future, it won't break ABI (see 3dd344ea84e122f791ab).

Thanks!

Quoted 7 lines
Quoted 4 lines

and we should take care of the return value of q->enqueue, because we can know what happens in the qdisc queue(not necessary to work with qdisc:dequeue).
and we can use a tracepoint filter for the return value too.

Disagree. Because we really have no interest in dropped packets. Even if we really do, we could trace kfree_skb(), not really here.

The qdisc returns not only the NET_XMIT_DROP, right ? skbprio_enqueue, sfq_enqueue and red_enqueue may return the NET_XMIT_CN.

Quoted 9 lines
Quoted 2 lines

we should introduce a new function to instead of now codes, that may make the codes clean. Please review my patch for more info.

Just 3 lines of code, it is totally personal taste.

Quoted 1 line

https://patchwork.kernel.org/project/netdevbpf/patch/[email protected]/

I did review it. Like I said, %p does not work. Have you tested your patches? ;)

Yes, I tested them, I didn't find the error. my patch is based on commit id 89212e160b81e778f829b89743570665810e3b13

Quoted 1 line

Thanks.

Signature · 1 line
Best regards, Tonghao
author
#4 lore raw
Quoted 69 lines · Tonghao Zhang
Quoted 58 lines · Cong Wang
Quoted 47 lines · Tonghao Zhang
Quoted 45 lines · Cong Wang

From: Qitao Xu <[email protected]>

Tracepoint trace_qdisc_enqueue() is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric to trace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate than trace_net_dev_queue(), because we already successfully enqueue the packet at that point.

Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.

Cc Jamal Hadi Salim <[email protected]>
Cc Jiri Pirko <[email protected]>
---
 include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++
 net/core/dev.c               |  9 +++++++++
 2 files changed, 35 insertions(+)
include/trace/events/qdisc.h +15 -0
index 58209557cb3a..c3006c6b4a87 100644
@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue,
                  __entry->txq_state, __entry->packets, __entry->skbaddr )
 );
 
+TRACE_EVENT(qdisc_enqueue,
+
+       TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb),
+
+       TP_ARGS(qdisc, txq, skb),
+
+       TP_STRUCT__entry(
+               __field(struct Qdisc *, qdisc)
+               __field(void *, skbaddr)
+               __field(int, ifindex)
+               __field(u32, handle)
+               __field(u32, parent)
+       ),
+
+       TP_fast_assign(
Quoted 8 lines

include/trace/events/qdisc.h TRACE_EVENT(qdisc_dequeue, · line 69

Quoted 6 lines
include/trace/events/qdisc.h +6 -0
+               __entry->qdisc = qdisc;
+               __entry->skbaddr = skb;
+               __entry->ifindex = txq->dev ? txq->dev->ifindex : 0;
+               __entry->handle  = qdisc->handle;
+               __entry->parent  = qdisc->parent;
+       ),

Hi qitao, cong
Why not support the txq, we get more info from txq.

Quoted 3 lines

Because we only want to calculate queueing latency, not anything else. If you need it, you are welcome to add anything reasonable in the future, it won't break ABI (see 3dd344ea84e122f791ab).

Thanks!

Quoted 5 lines
Quoted 4 lines

and we should take care of the return value of q->enqueue, because we can know what happens in the qdisc queue(not necessary to work with qdisc:dequeue).
and we can use a tracepoint filter for the return value too.

Quoted 5 lines
Quoted 2 lines

Disagree. Because we really have no interest in dropped packets. Even if we really do, we could trace kfree_skb(), not really here.

The qdisc returns not only the NET_XMIT_DROP, right ? skbprio_enqueue, sfq_enqueue and red_enqueue may return the NET_XMIT_CN.

Sure, in that case a different packet is dropped, once again you can trace it with kfree_skb() if you want. What's the problem?

Quoted 4 lines
Quoted 4 lines
Quoted 3 lines

we should introduce a new function to instead of now codes, that may make the codes clean. Please review my patch for more info.

Quoted 8 lines
Quoted 6 lines

Just 3 lines of code, it is totally personal taste.

Quoted 1 line

https://patchwork.kernel.org/project/netdevbpf/patch/[email protected]/

I did review it. Like I said, %p does not work. Have you tested your patches? ;)

Yes, I tested them, I didn't find the error. my patch is based on commit id 89212e160b81e778f829b89743570665810e3b13

I seriously doubt it, because we actually used %p in the beginning too and got the same address for two different packets, this is why we have to move to %px. It is 100% reproducible, so it probably means you didn't test it at all.

Thanks.

#5 lore raw
Quoted 71 lines · Cong Wang
Quoted 69 lines · Tonghao Zhang
Quoted 58 lines · Cong Wang
Quoted 47 lines · Tonghao Zhang
Quoted 45 lines · Cong Wang

From: Qitao Xu <[email protected]>

Tracepoint trace_qdisc_enqueue() is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric to trace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate than trace_net_dev_queue(), because we already successfully enqueue the packet at that point.

Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.

Cc Jamal Hadi Salim <[email protected]>
Cc Jiri Pirko <[email protected]>
---
 include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++
 net/core/dev.c               |  9 +++++++++
 2 files changed, 35 insertions(+)
include/trace/events/qdisc.h +15 -0
index 58209557cb3a..c3006c6b4a87 100644
@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue,
                  __entry->txq_state, __entry->packets, __entry->skbaddr )
 );
 
+TRACE_EVENT(qdisc_enqueue,
+
+       TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb),
+
+       TP_ARGS(qdisc, txq, skb),
+
+       TP_STRUCT__entry(
+               __field(struct Qdisc *, qdisc)
+               __field(void *, skbaddr)
+               __field(int, ifindex)
+               __field(u32, handle)
+               __field(u32, parent)
+       ),
+
+       TP_fast_assign(
Quoted 8 lines

include/trace/events/qdisc.h TRACE_EVENT(qdisc_dequeue, · line 69

Quoted 6 lines
include/trace/events/qdisc.h +6 -0
+               __entry->qdisc = qdisc;
+               __entry->skbaddr = skb;
+               __entry->ifindex = txq->dev ? txq->dev->ifindex : 0;
+               __entry->handle  = qdisc->handle;
+               __entry->parent  = qdisc->parent;
+       ),

Hi qitao, cong
Why not support the txq, we get more info from txq.

Quoted 3 lines

Because we only want to calculate queueing latency, not anything else. If you need it, you are welcome to add anything reasonable in the future, it won't break ABI (see 3dd344ea84e122f791ab).

Thanks!

Quoted 5 lines
Quoted 4 lines

and we should take care of the return value of q->enqueue, because we can know what happens in the qdisc queue(not necessary to work with qdisc:dequeue).
and we can use a tracepoint filter for the return value too.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

Disagree. Because we really have no interest in dropped packets. Even if we really do, we could trace kfree_skb(), not really here.

The qdisc returns not only the NET_XMIT_DROP, right ? skbprio_enqueue, sfq_enqueue and red_enqueue may return the NET_XMIT_CN.

Sure, in that case a different packet is dropped, once again you can trace it with kfree_skb() if you want. What's the problem?

It's ok, but we can make it better. Yunsheng Lin may have explained why?

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 3 lines

we should introduce a new function to instead of now codes, that may make the codes clean. Please review my patch for more info.

Just 3 lines of code, it is totally personal taste.

Quoted 1 line

https://patchwork.kernel.org/project/netdevbpf/patch/[email protected]/

I did review it. Like I said, %p does not work. Have you tested your patches? ;)

Quoted 7 lines
Quoted 2 lines

Yes, I tested them, I didn't find the error. my patch is based on commit id 89212e160b81e778f829b89743570665810e3b13

I seriously doubt it, because we actually used %p in the beginning too and got the same address for two different packets, this is why we have to move to %px. It is 100% reproducible, so it probably means you didn't test it at all.

I use iperf to send the packets, not found the same address really

          <idle>-0       [043] ..s.    62.493634: qdisc_enqueue:
enqueue ifindex=2 qdisc handle=0x0 parent=0x2C
skbaddr=00000000a40f93fb ret=0
          <idle>-0       [043] ..s.    62.494641: qdisc_enqueue:
enqueue ifindex=2 qdisc handle=0x0 parent=0x20
skbaddr=00000000c3c53e95 ret=0
          <idle>-0       [014] ..s.    64.473877: qdisc_enqueue:
enqueue ifindex=2 qdisc handle=0x0 parent=0x20
skbaddr=00000000ad610424 ret=0
          <idle>-0       [014] ..s.    64.473896: qdisc_enqueue:

enqueue ifindex=2 qdisc handle=0x0 parent=0x20
skbaddr=00000000112a562d ret=0

Quoted 2 lines

Thanks.

Signature · 1 line
Best regards, Tonghao
author
#6 lore raw
Quoted 5 lines · Tonghao Zhang
Quoted 2 lines · Cong Wang

Sure, in that case a different packet is dropped, once again you can trace it with kfree_skb() if you want. What's the problem?

It's ok, but we can make it better. Yunsheng Lin may have explained why?

Why it is better to trace dropped packets both in enqueue and in kfree_skb()? I fail to see it. You are just asking for duplications. If you do not see it by yourself, it means you don't understand or need it at all. ;)

Thanks.

#7 lore raw
Quoted 9 lines · Cong Wang
Quoted 5 lines · Tonghao Zhang
Quoted 2 lines · Cong Wang

Sure, in that case a different packet is dropped, once again you can trace it with kfree_skb() if you want. What's the problem?

It's ok, but we can make it better. Yunsheng Lin may have explained why?

Why it is better to trace dropped packets both in enqueue and in kfree_skb()?

I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and kfree_skb()
If so, we must match when the packet is dropped and what packets. if we use the return value in trace_qdisc_requeue. It
is easy to know what happened(when, where, what packets were dropped ).

Quoted 2 lines

I fail to see it. You are just asking for duplications. If you do not see it by yourself, it means you don't understand or need it at all. ;)

I added the tracepoint in centos 8 4.18 kernel version in our servers for a long time.

Quoted 1 line

Thanks.

Signature · 1 line
Best regards, Tonghao
author
#8 lore raw
Quoted 5 lines · Tonghao Zhang
Quoted 3 lines · Cong Wang
Quoted 1 line · Tonghao Zhang
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines · Cong Wang
Quoted 2 lines

Sure, in that case a different packet is dropped, once again you can trace it with kfree_skb() if you want. What's the problem?

It's ok, but we can make it better. Yunsheng Lin may have explained why?

Why it is better to trace dropped packets both in enqueue and in kfree_skb()?

I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and kfree_skb()

This is wrong, packets can be dropped for other reasons too, tracing enqueue is clearly not sufficient even if you trace it unconditionally. For a quick example, codel drops packets at dequeue rather than enqueue. ;)

Quoted 3 lines

If so, we must match when the packet is dropped and what packets. if we use the return value in trace_qdisc_requeue. It
is easy to know what happened(when, where, what packets were dropped ).

I am afraid you have to watch the dropped packets.

Thanks.

#9 lore raw
Quoted 10 lines · Cong Wang
Quoted 8 lines · Tonghao Zhang
Quoted 6 lines · Cong Wang
Quoted 4 lines · Tonghao Zhang
Quoted 2 lines · Cong Wang

Sure, in that case a different packet is dropped, once again you can trace it with kfree_skb() if you want. What's the problem?

Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line

It's ok, but we can make it better. Yunsheng Lin may have explained why?

Why it is better to trace dropped packets both in enqueue and in kfree_skb()?

I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and kfree_skb()

This is wrong, packets can be dropped for other reasons too, tracing

no matter where the packet is dropped, we should allow user to know whether dropped in the enqueue. and the
what the return value.

Quoted 11 lines

enqueue is clearly not sufficient even if you trace it unconditionally. For a quick example, codel drops packets at dequeue rather than enqueue. ;)

Quoted 3 lines

If so, we must match when the packet is dropped and what packets. if we use the return value in trace_qdisc_requeue. It
is easy to know what happened(when, where, what packets were dropped ).

I am afraid you have to watch the dropped packets.

Thanks.

Signature · 1 line
Best regards, Tonghao
author
#10 lore raw
Quoted 14 lines · Tonghao Zhang
Quoted 12 lines · Cong Wang
Quoted 10 lines · Tonghao Zhang
Quoted 8 lines · Cong Wang
Quoted 5 lines · Tonghao Zhang
Quoted 2 lines · Cong Wang

Sure, in that case a different packet is dropped, once again you can trace it with kfree_skb() if you want. What's the problem?

It's ok, but we can make it better. Yunsheng Lin may have explained why?

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line

Why it is better to trace dropped packets both in enqueue and in kfree_skb()?

I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and kfree_skb()

This is wrong, packets can be dropped for other reasons too, tracing

no matter where the packet is dropped, we should allow user to know whether dropped in the enqueue. and the
what the return value.

Again you can know it by kfree_skb(). And you can not avoid kfree_skb() no matter how you change enqueue. So, I don't see your argument of saving kfree_skb() makes any sense here.

Thanks.

#11 lore raw
Quoted 20 lines · Cong Wang
Quoted 18 lines · Tonghao Zhang
Quoted 16 lines · Cong Wang
Quoted 13 lines · Tonghao Zhang
Quoted 9 lines · Cong Wang
Quoted 5 lines · Tonghao Zhang
Quoted 2 lines · Cong Wang

Sure, in that case a different packet is dropped, once again you can trace it with kfree_skb() if you want. What's the problem?

It's ok, but we can make it better. Yunsheng Lin may have explained why?

Why it is better to trace dropped packets both in enqueue and in kfree_skb()?

I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and kfree_skb()

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

This is wrong, packets can be dropped for other reasons too, tracing

no matter where the packet is dropped, we should allow user to know whether dropped in the enqueue. and the
what the return value.

Again you can know it by kfree_skb(). And you can not avoid kfree_skb() no matter how you change enqueue. So, I don't see your

No, If I know what value returned for specified qdisc , I can know what happened, not necessarily kfree_skb()

Quoted 3 lines

argument of saving kfree_skb() makes any sense here.

Thanks.

Signature · 1 line
Best regards, Tonghao
author
#12 lore raw
Quoted 23 lines · Tonghao Zhang
Quoted 21 lines · Cong Wang
Quoted 19 lines · Tonghao Zhang
Quoted 17 lines · Cong Wang
Quoted 5 lines · Tonghao Zhang
Quoted 3 lines · Cong Wang
Quoted 1 line · Tonghao Zhang
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines · Cong Wang
Quoted 2 lines

Sure, in that case a different packet is dropped, once again you can trace it with kfree_skb() if you want. What's the problem?

It's ok, but we can make it better. Yunsheng Lin may have explained why?

Why it is better to trace dropped packets both in enqueue and in kfree_skb()?

I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and kfree_skb()

This is wrong, packets can be dropped for other reasons too, tracing

Quoted 8 lines
Quoted 6 lines
Quoted 3 lines

no matter where the packet is dropped, we should allow user to know whether dropped in the enqueue. and the
what the return value.

Again you can know it by kfree_skb(). And you can not avoid kfree_skb() no matter how you change enqueue. So, I don't see your

No, If I know what value returned for specified qdisc , I can know what happened, not necessarily kfree_skb()

This is wrong. You have to trace dropped packets because you need to know when to delete the key (skb address) from the hashtable you use to calculate the latency. You save the key on enqueue and remove it on both dequeue and kfree_skb, the only difference is you only need to calculate the timestamp difference for the former.

Thanks.

#13 lore raw
Quoted 31 lines · Cong Wang
Quoted 29 lines · Tonghao Zhang
Quoted 27 lines · Cong Wang
Quoted 14 lines · Tonghao Zhang
Quoted 12 lines · Cong Wang
Quoted 10 lines · Tonghao Zhang
Quoted 8 lines · Cong Wang
Quoted 5 lines · Tonghao Zhang
Quoted 2 lines · Cong Wang

Sure, in that case a different packet is dropped, once again you can trace it with kfree_skb() if you want. What's the problem?

It's ok, but we can make it better. Yunsheng Lin may have explained why?

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line

Why it is better to trace dropped packets both in enqueue and in kfree_skb()?

I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and kfree_skb()

This is wrong, packets can be dropped for other reasons too, tracing

no matter where the packet is dropped, we should allow user to know whether dropped in the enqueue. and the
what the return value.

Again you can know it by kfree_skb(). And you can not avoid kfree_skb() no matter how you change enqueue. So, I don't see your

Quoted 7 lines
Quoted 2 lines

No, If I know what value returned for specified qdisc , I can know what happened, not necessarily kfree_skb()

This is wrong. You have to trace dropped packets because you need to know when to delete the key (skb address) from the hashtable you use to calculate the latency. You save the key on enqueue and remove it on both dequeue and kfree_skb, the only difference is you

No, we can set the timestamp or cb in skb in enqueue and check them in dequeue. we may not use the hashtable.
If we use hashtable, we still can check the return value, save them to hashtable or not.

Quoted 3 lines

only need to calculate the timestamp difference for the former.

Thanks.

Signature · 1 line
Best regards, Tonghao
author
#14 lore raw
Quoted 36 lines · Tonghao Zhang
Quoted 34 lines · Cong Wang
Quoted 31 lines · Tonghao Zhang
Quoted 20 lines · Cong Wang
Quoted 18 lines · Tonghao Zhang
Quoted 16 lines · Cong Wang
Quoted 13 lines · Tonghao Zhang
Quoted 9 lines · Cong Wang
Quoted 5 lines · Tonghao Zhang
Quoted 2 lines · Cong Wang

Sure, in that case a different packet is dropped, once again you can trace it with kfree_skb() if you want. What's the problem?

It's ok, but we can make it better. Yunsheng Lin may have explained why?

Why it is better to trace dropped packets both in enqueue and in kfree_skb()?

I mean we can use one tracepoint to know what happened in the queue, not necessary to trace enqueue and kfree_skb()

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

This is wrong, packets can be dropped for other reasons too, tracing

no matter where the packet is dropped, we should allow user to know whether dropped in the enqueue. and the
what the return value.

Again you can know it by kfree_skb(). And you can not avoid kfree_skb() no matter how you change enqueue. So, I don't see your

No, If I know what value returned for specified qdisc , I can know what happened, not necessarily kfree_skb()

Quoted 6 lines
Quoted 4 lines

This is wrong. You have to trace dropped packets because you need to know when to delete the key (skb address) from the hashtable you use to calculate the latency. You save the key on enqueue and remove it on both dequeue and kfree_skb, the only difference is you

No, we can set the timestamp or cb in skb in enqueue and check them in dequeue. we may not use the hashtable.

Are you sure this is safe?? How do you ensure we have enough space in skb->cb[]? More importantly, why do we even modify skb in tracepoint?

Quoted 2 lines

If we use hashtable, we still can check the return value, save them to hashtable or not.

How many times do I have to repeat the return value of enqueue is not sufficient even if you trace it? Can't you just look at codel_dequeue() and tell me how a drop at dequeue can be reflected to enqueue?

Thanks.

#15 lore raw
In reply to Cong Wang
Quoted 71 lines · Cong Wang

From: Qitao Xu <[email protected]>

Tracepoint trace_qdisc_enqueue() is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric to trace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate than trace_net_dev_queue(), because we already successfully enqueue the packet at that point.

Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.

Cc Jamal Hadi Salim <[email protected]>
Cc Jiri Pirko <[email protected]>
---
 include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++
 net/core/dev.c               |  9 +++++++++
 2 files changed, 35 insertions(+)
include/trace/events/qdisc.h +26 -0
index 58209557cb3a..c3006c6b4a87 100644
@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue,
 		  __entry->txq_state, __entry->packets, __entry->skbaddr )
 );
 
+TRACE_EVENT(qdisc_enqueue,
+
+	TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb),
+
+	TP_ARGS(qdisc, txq, skb),
+
+	TP_STRUCT__entry(
+		__field(struct Qdisc *, qdisc)
+		__field(void *,	skbaddr)
+		__field(int, ifindex)
+		__field(u32, handle)
+		__field(u32, parent)
+	),
+
+	TP_fast_assign(
+		__entry->qdisc = qdisc;
+		__entry->skbaddr = skb;
+		__entry->ifindex = txq->dev ? txq->dev->ifindex : 0;
+		__entry->handle	 = qdisc->handle;
+		__entry->parent	 = qdisc->parent;
+	),
+
+	TP_printk("enqueue ifindex=%d qdisc handle=0x%X parent=0x%X skbaddr=%px",
+		  __entry->ifindex, __entry->handle, __entry->parent, __entry->skbaddr)
+);
+
 TRACE_EVENT(qdisc_reset,
 
 	TP_PROTO(struct Qdisc *q),
net/core/dev.c +1 -0
index c253c2aafe97..20b9376de301 100644
@@ -131,6 +131,7 @@
 #include <trace/events/napi.h>
 #include <trace/events/net.h>
 #include <trace/events/skb.h>
+#include <trace/events/qdisc.h>
 #include <linux/inetdevice.h>
 #include <linux/cpu_rmap.h>
 #include <linux/static_key.h>

net/core/dev.c static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, · line 3868

Quoted 4 lines
net/core/dev.c +1 -0
@@ -3864,6 +3865,8 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
 			if (unlikely(!nolock_qdisc_is_empty(q))) {
 				rc = q->enqueue(skb, q, &to_free) &
 					NET_XMIT_MASK;
+				if (rc == NET_XMIT_SUCCESS)

If NET_XMIT_CN is returned, the skb seems to be enqueued too?

Also instead of checking the rc before calling the trace_*, maybe it make more sense to add the rc to the tracepoint, so that the checking is avoided, and we are able to tell the enqueuing result of a specific skb from that tracepoint too.

net/core/dev.c static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, · line 3869

Quoted 1 line
net/core/dev.c +1 -0
+					trace_qdisc_enqueue(q, txq, skb);

Does it make sense to wrap the about to something like:

int sch_enqueue(....)

{
	rc = q->enqueue(skb, q, &to_free)..
	....
	trace_qdisc_enqueue(q, txq, skb);
}

So that the below code can reuse that wrapper too.

Quoted 24 lines
net/core/dev.c +6 -0
 				__qdisc_run(q);
 				qdisc_run_end(q);
 
@@ -3880,6 +3883,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
 		}
 
 		rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
+		if (rc == NET_XMIT_SUCCESS)
+			trace_qdisc_enqueue(q, txq, skb);
+
 		qdisc_run(q);
 
 no_lock_out:
@@ -3924,6 +3930,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
 		rc = NET_XMIT_SUCCESS;
 	} else {
 		rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
+		if (rc == NET_XMIT_SUCCESS)
+			trace_qdisc_enqueue(q, txq, skb);
+
 		if (qdisc_run_begin(q)) {
 			if (unlikely(contended)) {
 				spin_unlock(&q->busylock);
#16 lore raw
Quoted 80 lines · Yunsheng Lin
Quoted 71 lines · Cong Wang

From: Qitao Xu <[email protected]>

Tracepoint trace_qdisc_enqueue() is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric to trace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate than trace_net_dev_queue(), because we already successfully enqueue the packet at that point.

Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.

Cc Jamal Hadi Salim <[email protected]>
Cc Jiri Pirko <[email protected]>
---
 include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++
 net/core/dev.c               |  9 +++++++++
 2 files changed, 35 insertions(+)
include/trace/events/qdisc.h +26 -0
index 58209557cb3a..c3006c6b4a87 100644
@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue,
                __entry->txq_state, __entry->packets, __entry->skbaddr )
 );
 
+TRACE_EVENT(qdisc_enqueue,
+
+     TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb),
+
+     TP_ARGS(qdisc, txq, skb),
+
+     TP_STRUCT__entry(
+             __field(struct Qdisc *, qdisc)
+             __field(void *, skbaddr)
+             __field(int, ifindex)
+             __field(u32, handle)
+             __field(u32, parent)
+     ),
+
+     TP_fast_assign(
+             __entry->qdisc = qdisc;
+             __entry->skbaddr = skb;
+             __entry->ifindex = txq->dev ? txq->dev->ifindex : 0;
+             __entry->handle  = qdisc->handle;
+             __entry->parent  = qdisc->parent;
+     ),
+
+     TP_printk("enqueue ifindex=%d qdisc handle=0x%X parent=0x%X skbaddr=%px",
+               __entry->ifindex, __entry->handle, __entry->parent, __entry->skbaddr)
+);
+
 TRACE_EVENT(qdisc_reset,
 
      TP_PROTO(struct Qdisc *q),
net/core/dev.c +1 -0
index c253c2aafe97..20b9376de301 100644
@@ -131,6 +131,7 @@
 #include <trace/events/napi.h>
 #include <trace/events/net.h>
 #include <trace/events/skb.h>
+#include <trace/events/qdisc.h>
 #include <linux/inetdevice.h>
 #include <linux/cpu_rmap.h>
 #include <linux/static_key.h>

net/core/dev.c static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, · line 3868

Quoted 4 lines
net/core/dev.c +1 -0
@@ -3864,6 +3865,8 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
                      if (unlikely(!nolock_qdisc_is_empty(q))) {
                              rc = q->enqueue(skb, q, &to_free) &
                                      NET_XMIT_MASK;
+                             if (rc == NET_XMIT_SUCCESS)

If NET_XMIT_CN is returned, the skb seems to be enqueued too?

Quoted 4 lines

Also instead of checking the rc before calling the trace_*, maybe it make more sense to add the rc to the tracepoint, so that the checking is avoided, and we are able to tell the enqueuing result of a specific skb from that tracepoint too.

Yes, I will fix it.

Quoted 10 lines

net/core/dev.c static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, · line 3869

Quoted 1 line
net/core/dev.c +1 -0
+                                     trace_qdisc_enqueue(q, txq, skb);

Does it make sense to wrap the about to something like:

int sch_enqueue(....)

{
        rc = q->enqueue(skb, q, &to_free)..
        ....
        trace_qdisc_enqueue(q, txq, skb);
}

Yes, I agree, my patch uses qdisc_enqueue_skb, because __dev_xmit_skb invoke the qdisc_xxx api.

https://patchwork.kernel.org/project/netdevbpf/patch/[email protected]/

Quoted 26 lines

So that the below code can reuse that wrapper too.

Quoted 24 lines
net/core/dev.c +6 -0
                              __qdisc_run(q);
                              qdisc_run_end(q);
 
@@ -3880,6 +3883,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
              }
 
              rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
+             if (rc == NET_XMIT_SUCCESS)
+                     trace_qdisc_enqueue(q, txq, skb);
+
              qdisc_run(q);
 
 no_lock_out:
@@ -3924,6 +3930,9 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
              rc = NET_XMIT_SUCCESS;
      } else {
              rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
+             if (rc == NET_XMIT_SUCCESS)
+                     trace_qdisc_enqueue(q, txq, skb);
+
              if (qdisc_run_begin(q)) {
                      if (unlikely(contended)) {
                              spin_unlock(&q->busylock);
Signature · 1 line
Best regards, Tonghao
author
#17 lore raw
In reply to Yunsheng Lin
Quoted 71 lines · Yunsheng Lin
Quoted 69 lines · Cong Wang

From: Qitao Xu <[email protected]>

Tracepoint trace_qdisc_enqueue() is introduced to trace skb at the entrance of TC layer on TX side. This is kinda symmetric to trace_qdisc_dequeue(), and together they can be used to calculate the packet queueing latency. It is more accurate than trace_net_dev_queue(), because we already successfully enqueue the packet at that point.

Note, trace ring buffer is only accessible to privileged users, it is safe to use %px to print a real kernel address here.

Cc Jamal Hadi Salim <[email protected]>
Cc Jiri Pirko <[email protected]>
---
 include/trace/events/qdisc.h | 26 ++++++++++++++++++++++++++
 net/core/dev.c               |  9 +++++++++
 2 files changed, 35 insertions(+)
include/trace/events/qdisc.h +26 -0
index 58209557cb3a..c3006c6b4a87 100644
@@ -46,6 +46,32 @@ TRACE_EVENT(qdisc_dequeue,
                __entry->txq_state, __entry->packets, __entry->skbaddr )
 );
 
+TRACE_EVENT(qdisc_enqueue,
+
+     TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq, struct sk_buff *skb),
+
+     TP_ARGS(qdisc, txq, skb),
+
+     TP_STRUCT__entry(
+             __field(struct Qdisc *, qdisc)
+             __field(void *, skbaddr)
+             __field(int, ifindex)
+             __field(u32, handle)
+             __field(u32, parent)
+     ),
+
+     TP_fast_assign(
+             __entry->qdisc = qdisc;
+             __entry->skbaddr = skb;
+             __entry->ifindex = txq->dev ? txq->dev->ifindex : 0;
+             __entry->handle  = qdisc->handle;
+             __entry->parent  = qdisc->parent;
+     ),
+
+     TP_printk("enqueue ifindex=%d qdisc handle=0x%X parent=0x%X skbaddr=%px",
+               __entry->ifindex, __entry->handle, __entry->parent, __entry->skbaddr)
+);
+
 TRACE_EVENT(qdisc_reset,
 
      TP_PROTO(struct Qdisc *q),
net/core/dev.c +1 -0
index c253c2aafe97..20b9376de301 100644
@@ -131,6 +131,7 @@
 #include <trace/events/napi.h>
 #include <trace/events/net.h>
 #include <trace/events/skb.h>
+#include <trace/events/qdisc.h>
 #include <linux/inetdevice.h>
 #include <linux/cpu_rmap.h>
Quoted 8 lines

net/core/dev.c static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, · line 3868

Quoted 6 lines
net/core/dev.c +1 -0
 #include <linux/static_key.h>
@@ -3864,6 +3865,8 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
                      if (unlikely(!nolock_qdisc_is_empty(q))) {
                              rc = q->enqueue(skb, q, &to_free) &
                                      NET_XMIT_MASK;
+                             if (rc == NET_XMIT_SUCCESS)

If NET_XMIT_CN is returned, the skb seems to be enqueued too?

Sure. See the other reply from on why dropped packets are not interesting here.

Quoted 5 lines

Also instead of checking the rc before calling the trace_*, maybe it make more sense to add the rc to the tracepoint, so that the checking is avoided, and we are able to tell the enqueuing result of a specific skb from that tracepoint too.

Totally disagree, because trace_qdisc_dequeue() is only called for successful cases too (see dequeue_skb()), it does not make sense to let them be different.

Quoted 4 lines

net/core/dev.c static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, · line 3869

Quoted 2 lines
net/core/dev.c +1 -0
+                                     trace_qdisc_enqueue(q, txq, skb);

Does it make sense to wrap the about to something like:

Nope. Because ->enqueue() is called by lower layer qdisc's too, but here we only want to track root, aka, entrance of TC. I know this may be confusing, please blame trace_qdisc_dequeue() which only tracks the exit. ;)

Thanks.