tracing updates
Subsystems (3)
Ingo,
Please pull the perf/core branch that can be found at:
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/random-tracing.git
perf/core
Thanks,
Frederic
---
Koki Sanagi (3):
netdev: Add tracepoints to netdev layer
skb: Add tracepoints to freeing skb
perf: Add a script to show packets processing
Lai Jiangshan (1):
irq: Add tracepoint to softirq_raise
Neil Horman (1):
napi: Convert trace_napi_poll to TRACE_EVENT
include/linux/interrupt.h | 8 +- include/trace/events/irq.h | 26 +- include/trace/events/napi.h | 25 +- include/trace/events/net.h | 82 ++++ include/trace/events/skb.h | 17 + net/core/datagram.c | 1 + net/core/dev.c | 8 + net/core/net-traces.c | 1 + net/core/skbuff.c | 1 + tools/perf/scripts/python/bin/netdev-times-record | 8 + tools/perf/scripts/python/bin/netdev-times-report | 5 + tools/perf/scripts/python/netdev-times.py | 464 +++++++++++++++++++++ 12 files changed, 641 insertions(+), 5 deletions(-)
PATCH 1/5 irq: Add tracepoint to softirq_raise
From: Lai Jiangshan <[email protected]>
Add a tracepoint for tracing when softirq action is raised.
This and the existing tracepoints complete softirq's tracepoints:softirq_raise, softirq_entry and softirq_exit.
And when this tracepoint is used in combination with the softirq_entry tracepoint we can determine the softirq raise latency.
LKML-Reference: <[email protected]>
--- include/linux/interrupt.h | 8 +++++++- include/trace/events/irq.h | 26 ++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-)
include/linux/interrupt.h +7 -1
@@ -18,6 +18,7 @@#include <asm/atomic.h> #include <asm/ptrace.h> #include <asm/system.h> +#include <trace/events/irq.h> /* * These correspond to the IORESOURCE_IRQ_* defines in
@@ -407,7 +408,12 @@ asmlinkage void do_softirq(void); asmlinkage void __do_softirq(void);extern void open_softirq (int nr,void (*action)(struct softirq_action *));extern void softirq_init (void ); -#define __raise_softirq_irqoff (nr) do { or_softirq_pending(1UL << (nr)); } while (0) +static inline void __raise_softirq_irqoff (unsigned int nr) +{ +trace_softirq_raise ((struct softirq_action *)(unsigned long )nr,NULL ); +or_softirq_pending (1UL << nr); +} +extern void raise_softirq_irqoff (unsigned int nr);extern void raise_softirq (unsigned int nr);extern void wakeup_softirqd (void );
include/trace/events/irq.h +24 -2
@@ -5,7 +5,9 @@#define _TRACE_IRQ_H#include <linux/tracepoint.h> -#include <linux/interrupt.h> + +struct irqaction ; +struct softirq_action ;#define softirq_name (sirq) { sirq##_SOFTIRQ, #sirq }#define show_softirq_name (val) \
@@ -93,7 +95,10 @@ DECLARE_EVENT_CLASS(softirq, ),TP_fast_assign ( - __entry->vec = (int)(h - vec); + if (vec) + __entry->vec = (int)(h - vec); + else + __entry->vec = (int)(long)h; ), TP_printk("vec=%d [action=%s]", __entry->vec,
@@ -136,6 +141,23 @@ DEFINE_EVENT(softirq, softirq_exit,TP_ARGS (h, vec) ); +/** +* softirq_raise - called immediately when a softirq is raised +* @h: pointer to struct softirq_action +* @vec: pointer to first struct softirq_action in softirq_vec array +* +* The @h parameter contains a pointer to the softirq vector number which is +* raised. @vec is NULL and it means @h includes vector number not +* softirq_action. When used in combination with the softirq_entry tracepoint +* we can determine the softirq raise latency. +*/ +DEFINE_EVENT (softirq, softirq_raise, + + TP_PROTO(struct softirq_action *h, struct softirq_action *vec), + +TP_ARGS (h, vec) +); +#endif /* _TRACE_IRQ_H */ /* This part must be outside protection */
Signature · 1 line
1.6.2.3
PATCH 2/5 napi: Convert trace_napi_poll to TRACE_EVENT
In reply to Frederic WeisbeckerFrom: Neil Horman <[email protected]>
This patch converts trace_napi_poll from DECLARE_EVENT to TRACE_EVENT to improve the usability of napi_poll tracepoint.
<idle>-0 [001] 241302.750777: napi_poll: napi poll on napi struct f6acc480 for device eth3
<idle>-0 [000] 241302.852389: napi_poll: napi poll on napi struct f5d0d70c for device eth1The original patch is below:
http://marc.info/?l=linux-kernel&m=126021713809450&w=2
[ [email protected]: And add a fix by Steven Rostedt:
http://marc.info/?l=linux-kernel&m=126150506519173&w=2 ]
LKML-Reference: <[email protected]>
--- include/trace/events/napi.h | 25 +++++++++++++++++++++++-- 1 files changed, 23 insertions(+), 2 deletions(-)
include/trace/events/napi.h +23 -2
@@ -6,10 +6,31 @@#include <linux/netdevice.h> #include <linux/tracepoint.h> +#include <linux/ftrace.h> + +#define NO_DEV "(no_device)" + +TRACE_EVENT (napi_poll, -DECLARE_TRACE (napi_poll, TP_PROTO(struct napi_struct *napi), -TP_ARGS (napi)); + +TP_ARGS (napi), + +TP_STRUCT__entry ( + __field( struct napi_struct *, napi) +__string ( dev_name, napi->dev ? napi->dev->name :NO_DEV ) + ), + +TP_fast_assign ( + __entry->napi = napi; + __assign_str(dev_name, napi->dev ? napi->dev->name : NO_DEV); + ), + +TP_printk ("napi poll on napi struct %p for device %s" , + __entry->napi,__get_str (dev_name)) +); + +#undef NO_DEV#endif /* _TRACE_NAPI_H_ */
Signature · 1 line
1.6.2.3
PATCH 3/5 netdev: Add tracepoints to netdev layer
In reply to Frederic WeisbeckerFrom: Koki Sanagi <[email protected]>
This patch adds tracepoint to dev_queue_xmit, dev_hard_start_xmit, netif_rx and netif_receive_skb. These tracepoints help you to monitor network driver's input/output.
<idle>-0 [001] 112447.902030: netif_rx: dev=eth1 skbaddr=f3ef0900 len=84
<idle>-0 [001] 112447.902039: netif_receive_skb: dev=eth1 skbaddr=f3ef0900 len=84
sshd-6828 [000] 112447.903257: net_dev_queue: dev=eth4 skbaddr=f3fca538 len=226
sshd-6828 [000] 112447.903260: net_dev_xmit: dev=eth4 skbaddr=f3fca538 len=226 rc=0LKML-Reference: <[email protected]>
--- include/trace/events/net.h | 82 ++++++++++++++++++++++++++++++++++++++++++++ net/core/dev.c | 6 +++ net/core/net-traces.c | 1 + 3 files changed, 89 insertions(+), 0 deletions(-) create mode 100644 include/trace/events/net.h
include/trace/events/net.h new file +82 -0
@@ -0,0 +1,82 @@ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM net + +#if !defined(_TRACE_NET_H) || defined(TRACE_HEADER_MULTI_READ ) +#define _TRACE_NET_H + +#include <linux/skbuff.h> +#include <linux/netdevice.h> +#include <linux/ip.h> +#include <linux/tracepoint.h> + +TRACE_EVENT (net_dev_xmit, + + TP_PROTO(struct sk_buff *skb, + int rc), + +TP_ARGS (skb, rc), + +TP_STRUCT__entry ( + __field( void *, skbaddr ) + __field( unsigned int, len ) + __field( int, rc ) +__string ( name, skb->dev->name ) + ), + +TP_fast_assign ( + __entry->skbaddr = skb; + __entry->len = skb->len; + __entry->rc = rc; + __assign_str(name, skb->dev->name); + ), + +TP_printk ("dev=%s skbaddr=%p len=%u rc=%d" , +__get_str (name), __entry->skbaddr, __entry->len, __entry->rc) +); + +DECLARE_EVENT_CLASS (net_dev_template, + + TP_PROTO(struct sk_buff *skb), + +TP_ARGS (skb), + +TP_STRUCT__entry ( + __field( void *, skbaddr ) + __field( unsigned int, len ) +__string ( name, skb->dev->name ) + ), + +TP_fast_assign ( + __entry->skbaddr = skb; + __entry->len = skb->len; + __assign_str(name, skb->dev->name); + ), + +TP_printk ("dev=%s skbaddr=%p len=%u" , +__get_str (name), __entry->skbaddr, __entry->len) +) + +DEFINE_EVENT (net_dev_template, net_dev_queue, + + TP_PROTO(struct sk_buff *skb), + +TP_ARGS (skb) +); + +DEFINE_EVENT (net_dev_template, netif_receive_skb, + + TP_PROTO(struct sk_buff *skb), + +TP_ARGS (skb) +); + +DEFINE_EVENT (net_dev_template, netif_rx, + + TP_PROTO(struct sk_buff *skb), + +TP_ARGS (skb) +); +#endif /* _TRACE_NET_H */ + +/* This part must be outside protection */ +#include <trace/define_trace.h>
net/core/dev.c +6 -0
@@ -128,6 +128,7 @@#include <linux/jhash.h> #include <linux/random.h> #include <trace/events/napi.h> +#include <trace/events/net.h> #include <linux/pci.h> #include "net-sysfs.h"
@@ -1978,6 +1979,7 @@ int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev, } rc = ops->ndo_start_xmit (skb, dev); +trace_net_dev_xmit (skb, rc);if (rc ==NETDEV_TX_OK )txq_trans_update (txq);return rc;
@@ -1998,6 +2000,7 @@ gso:skb_dst_drop (nskb); rc = ops->ndo_start_xmit (nskb, dev); +trace_net_dev_xmit (nskb, rc);if (unlikely (rc !=NETDEV_TX_OK )) {if (rc & ~NETDEV_TX_MASK ) goto out_kfree_gso_skb;
@@ -2186,6 +2189,7 @@ int dev_queue_xmit(struct sk_buff *skb)#ifdef CONFIG_NET_CLS_ACT skb->tc_verd =SET_TC_AT (skb->tc_verd,AT_EGRESS );#endif +trace_net_dev_queue (skb);if (q->enqueue) { rc =__dev_xmit_skb (skb, q, dev, txq); goto out;
@@ -2512,6 +2516,7 @@ int netif_rx(struct sk_buff *skb)if (netdev_tstamp_prequeue)net_timestamp_check (skb); +trace_netif_rx (skb);#ifdef CONFIG_RPS {struct rps_dev_flow voidflow, *rflow = &voidflow;
@@ -2828,6 +2833,7 @@ static int __netif_receive_skb(struct sk_buff *skb)if (!netdev_tstamp_prequeue)net_timestamp_check (skb); +trace_netif_receive_skb (skb);if (vlan_tx_tag_present (skb) &&vlan_hwaccel_do_receive (skb))return NET_RX_SUCCESS ;
net/core/net-traces.c +1 -0
@@ -26,6 +26,7 @@#define CREATE_TRACE_POINTS #include <trace/events/skb.h> +#include <trace/events/net.h> #include <trace/events/napi.h> EXPORT_TRACEPOINT_SYMBOL_GPL (kfree_skb);
Signature · 1 line
1.6.2.3
PATCH 4/5 skb: Add tracepoints to freeing skb
In reply to Frederic WeisbeckerFrom: Koki Sanagi <[email protected]>
This patch adds tracepoint to consume_skb and add trace_kfree_skb before __kfree_skb in skb_free_datagram_locked and net_tx_action. Combinating with tracepoint on dev_hard_start_xmit, we can check how long it takes to free transmitted packets. And using it, we can calculate how many packets driver had at that time. It is useful when a drop of transmitted packet is a problem.
sshd-6828 [000] 112689.258154: consume_skb: skbaddr=f2d99bb8
LKML-Reference: <[email protected]>
--- include/trace/events/skb.h | 17 +++++++++++++++++ net/core/datagram.c | 1 + net/core/dev.c | 2 ++ net/core/skbuff.c | 1 + 4 files changed, 21 insertions(+), 0 deletions(-)
include/trace/events/skb.h +17 -0
@@ -35,6 +35,23 @@ TRACE_EVENT(kfree_skb, __entry->skbaddr, __entry->protocol, __entry->location) ); +TRACE_EVENT (consume_skb, + + TP_PROTO(struct sk_buff *skb), + +TP_ARGS (skb), + +TP_STRUCT__entry ( + __field( void *, skbaddr ) + ), + +TP_fast_assign ( + __entry->skbaddr = skb; + ), + +TP_printk ("skbaddr=%p" , __entry->skbaddr) +); + TRACE_EVENT(skb_copy_datagram_iovec, TP_PROTO(const struct sk_buff *skb, int len),
net/core/datagram.c +1 -0
@@ -243,6 +243,7 @@ void skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb)unlock_sock_fast (sk, slow);/* skb is now orphaned, can be freed outside of locked section */ +trace_kfree_skb (skb, skb_free_datagram_locked);__kfree_skb (skb); }EXPORT_SYMBOL (skb_free_datagram_locked);
net/core/dev.c +2 -0
@@ -129,6 +129,7 @@#include <linux/random.h> #include <trace/events/napi.h> #include <trace/events/net.h> +#include <trace/events/skb.h> #include <linux/pci.h> #include "net-sysfs.h"
@@ -2576,6 +2577,7 @@ static void net_tx_action(struct softirq_action *h) clist = clist->next;WARN_ON (atomic_read (&skb->users)); +trace_kfree_skb (skb, net_tx_action);__kfree_skb (skb); } }
net/core/skbuff.c +1 -0
@@ -466,6 +466,7 @@ void consume_skb(struct sk_buff *skb)smp_rmb (); else if (likely(!atomic_dec_and_test(&skb->users))) return; +trace_consume_skb (skb );__kfree_skb (skb); }EXPORT_SYMBOL (consume_skb);
Signature · 1 line
1.6.2.3
PATCH 5/5 perf: Add a script to show packets processing
In reply to Frederic WeisbeckerFrom: Koki Sanagi <[email protected]>
Add a perf script which shows packets processing and processed time. It helps us to investigate networking or network devices.
If you want to use it, install perf and record perf.data like following.
If you set script, perf gathers records until it ends. If not, you must Ctrl-C to stop recording.
And if you want a report from record,
If you use some options, you can limit the output. Option is below.
tx: show only tx packets processing
rx: show only rx packets processing
dev=: show processing on this device
debug: work with debug mode. It shows buffer status.
For example, if you want to show received packets processing associated with eth4,
106133.171439sec cpu=0
irq_entry(+0.000msec irq=24:eth4)
|
softirq_entry(+0.006msec)
|
|---netif_receive_skb(+0.010msec skb=f2d15900 len=100)
| |
| skb_copy_datagram_iovec(+0.039msec 10291::10291)
| napi_poll_exit(+0.022msec eth4)
This perf script helps us to analyze the processing time of a transmit/receive sequence.
LKML-Reference: <[email protected]>
--- tools/perf/scripts/python/bin/netdev-times-record | 8 + tools/perf/scripts/python/bin/netdev-times-report | 5 + tools/perf/scripts/python/netdev-times.py | 464 +++++++++++++++++++++ 3 files changed, 477 insertions(+), 0 deletions(-) create mode 100644 tools/perf/scripts/python/bin/netdev-times-record create mode 100644 tools/perf/scripts/python/bin/netdev-times-report create mode 100644 tools/perf/scripts/python/netdev-times.py
tools/perf/scripts/python/bin/netdev-times-record new file +8 -0
@@ -0,0 +1,8 @@ +#!/bin/bash +perf record -a -e net:net_dev_xmit -e net:net_dev_queue \ + -e net:netif_receive_skb -e net:netif_rx \ + -e skb:consume_skb -e skb:kfree_skb \ + -e skb:skb_copy_datagram_iovec -e napi:napi_poll \ + -e irq:irq_handler_entry -e irq:irq_handler_exit \ + -e irq:softirq_entry -e irq:softirq_exit \ + -e irq:softirq_raise $@
tools/perf/scripts/python/bin/netdev-times-report new file +5 -0
@@ -0,0 +1,5 @@ +#!/bin/bash +# description: display a process of packet and processing time +# args: [tx] [rx] [dev=] [debug] + +perf trace -s ~/libexec/perf-core/scripts/python/netdev-times.py $@
tools/perf/scripts/python/netdev-times.py new file +464 -0
464 lines not on this page · raw message
Signature · 1 line
1.6.2.3
Quoted 23 lines · Frederic Weisbecker
Ingo,
Please pull the perf/core branch that can be found at:
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/random-tracing.git
perf/coreThanks,
Frederic
---Koki Sanagi (3):
netdev: Add tracepoints to netdev layer
skb: Add tracepoints to freeing skb
perf: Add a script to show packets processingLai Jiangshan (1):
irq: Add tracepoint tosoftirq_raiseNeil Horman (1):
napi: Converttrace_napi_polltoTRACE_EVENT
Quoted 13 lines
include/linux/interrupt.h | 8 +- include/trace/events/irq.h | 26 +- include/trace/events/napi.h | 25 +- include/trace/events/net.h | 82 ++++ include/trace/events/skb.h | 17 + net/core/datagram.c | 1 + net/core/dev.c | 8 + net/core/net-traces.c | 1 + net/core/skbuff.c | 1 + tools/perf/scripts/python/bin/netdev-times-record | 8 + tools/perf/scripts/python/bin/netdev-times-report | 5 + tools/perf/scripts/python/netdev-times.py | 464 +++++++++++++++++++++ 12 files changed, 641 insertions(+), 5 deletions(-)
Pulled, thanks a lot!
Ingo