Skip to contentSkip to conversation

net: remove static inline from dev_put()/dev_hold()

PATCH In review Tony Lu 3 participants 4 messages Nov 12, 2019 – Nov 16, 2019

Subsystems (3)
lore
Share & tools
lore

Tony Lu

author
#0 lore raw

PATCH v2 1/2 net: remove static inline from dev_put()/dev_hold()

This patch removes static inline from dev_put()/dev_hold() in order to help trace the pcpu_refcnt leak of net_device.

We have sufferred this kind of issue for several times during manipulating NIC between different net namespaces. It prints this log in dmesg:

unregister_netdevice: waiting for eth0 to become free. Usage count = 1

However, it is hard to find out who called and leaked refcnt in time. It only left the crime scene but few evidence. Once leaked, it is not safe to fix it up on the running host. We can't trace dev_put/dev_hold directly, for the functions are inlined and used wildly amoung modules. And this issue is common, there are tens of patches fixes net_device refcnt leak for various causes.

To trace the refcnt manipulating, this patch removes static inline from dev_put()/dev_hold(). We can use tools, such as eBPF with kprobe, to find out who holds but forgets to put refcnt. This will not be called frequently, so the overhead is limited.

---
v1->v2:
- provides two trace events and tracepoitns together in patch #2
- fix some typos in commit message

---
 include/linux/netdevice.h | 24 ++++--------------------
 net/core/dev.c            | 24 ++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 20 deletions(-)
include/linux/netdevice.h +4 -20
index c20f190b4c18..872d266c6da5 100644
@@ -3720,27 +3720,11 @@ extern unsigned int	netdev_budget_usecs;
 /* Called by rtnetlink.c:rtnl_unlock() */
 void netdev_run_todo(void);
 
-/**
- *	dev_put - release reference to device
- *	@dev: network device
- *
- * Release reference to device to allow it to be freed.
- */
-static inline void dev_put(struct net_device *dev)
-{
-	this_cpu_dec(*dev->pcpu_refcnt);
-}
+/* Release reference to device to allow it to be freed. */
+void dev_put(struct net_device *dev);
 
-/**
- *	dev_hold - get reference to device
- *	@dev: network device
- *
- * Hold reference to device to keep it from being freed.
- */
-static inline void dev_hold(struct net_device *dev)
-{
-	this_cpu_inc(*dev->pcpu_refcnt);
-}
+/* Hold reference to device to keep it from being freed. */
+void dev_hold(struct net_device *dev);
 
 /* Carrier loss detection, dial on demand. The functions netif_carrier_on
  * and _off may be called from IRQ context, but it is caller
net/core/dev.c +24 -0
index 99ac84ff398f..620fb3d6718a 100644
@@ -1294,6 +1294,30 @@ void netdev_notify_peers(struct net_device *dev)
 }
 EXPORT_SYMBOL(netdev_notify_peers);
 
+/**
+ *	dev_put - release reference to device
+ *	@dev: network device
+ *
+ * Release reference to device to allow it to be freed.
+ */
+void dev_put(struct net_device *dev)
+{
+	this_cpu_dec(*dev->pcpu_refcnt);
+}
+EXPORT_SYMBOL(dev_put);
+
+/**
+ *	dev_hold - get reference to device
+ *	@dev: network device
+ *
+ * Hold reference to device to keep it from being freed.
+ */
+void dev_hold(struct net_device *dev)
+{
+	this_cpu_inc(*dev->pcpu_refcnt);
+}
+EXPORT_SYMBOL(dev_hold);
+
 static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
 {
 	const struct net_device_ops *ops = dev->netdev_ops;
Signature · 1 line
2.24.0
author
#1 lore raw

PATCH v2 2/2 net: add trace events for net_device refcnt

The net_device refcnt leak is hard to trace and debug for now. We need the ability to know when and who manipulated the refcnt.

Adding the trace events for net_device pcpu_refcnt and also tracepoints in dev_put()/dev_hold(), provides the history of net_device refcnt inc and desc. With trace logs analysis, paring the put and hold history, we can find out who leaked.

---
 include/trace/events/net.h | 41 ++++++++++++++++++++++++++++++++++++++
 net/core/dev.c             |  4 ++++
 2 files changed, 45 insertions(+)
include/trace/events/net.h +41 -0
index 3b28843652d2..3bf6dd738882 100644
@@ -326,6 +326,47 @@ DEFINE_EVENT(net_dev_rx_exit_template, netif_receive_skb_list_exit,
 	TP_ARGS(ret)
 );
 
+DECLARE_EVENT_CLASS(net_dev_refcnt_template,
+
+	TP_PROTO(struct net_device *dev, void *location),
+
+	TP_ARGS(dev, location),
+
+	TP_STRUCT__entry(
+		__string(	name,		dev->name	)
+		__field(	int,		refcnt		)
+		__field(	void *,		location	)
+	),
+
+	TP_fast_assign(
+		int i, refcnt = 0;
+
+		for_each_possible_cpu(i)
+			refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
+
+		__assign_str(name, dev->name);
+		__entry->refcnt = refcnt;
+		__entry->location = location;
+	),
+
+	TP_printk("dev=%s refcnt=%d location=%p",
+		__get_str(name), __entry->refcnt, __entry->location)
+);
+
+DEFINE_EVENT(net_dev_refcnt_template, net_dev_put,
+
+	TP_PROTO(struct net_device *dev, void *location),
+
+	TP_ARGS(dev, location)
+);
+
+DEFINE_EVENT(net_dev_refcnt_template, net_dev_hold,
+
+	TP_PROTO(struct net_device *dev, void *location),
+
+	TP_ARGS(dev, location)
+);
+
 #endif /* _TRACE_NET_H */
 
 /* This part must be outside protection */
net/core/dev.c +4 -0
index 620fb3d6718a..163870a09984 100644
@@ -1302,6 +1302,8 @@ EXPORT_SYMBOL(netdev_notify_peers);
  */
 void dev_put(struct net_device *dev)
 {
+	trace_net_dev_put(dev, __builtin_return_address(0));
+
 	this_cpu_dec(*dev->pcpu_refcnt);
 }
 EXPORT_SYMBOL(dev_put);
@@ -1314,6 +1316,8 @@ EXPORT_SYMBOL(dev_put);
  */
 void dev_hold(struct net_device *dev)
 {
+	trace_net_dev_hold(dev, __builtin_return_address(0));
+
 	this_cpu_inc(*dev->pcpu_refcnt);
 }
 EXPORT_SYMBOL(dev_hold);
Signature · 1 line
2.24.0
#2 lore raw

Re: [PATCH v2 2/2] net: add trace events for net_device refcnt

Quoted 15 lines · Tony Lu
include/trace/events/net.h +7 -0
index 3b28843652d2..3bf6dd738882 100644
@@ -326,6 +326,47 @@ DEFINE_EVENT(net_dev_rx_exit_template, netif_receive_skb_list_exit,
 	TP_ARGS(ret)
 );
 
+DECLARE_EVENT_CLASS(net_dev_refcnt_template,
+
+	TP_PROTO(struct net_device *dev, void *location),
+
+	TP_ARGS(dev, location),
+
+	TP_STRUCT__entry(

include/trace/events/net.h DEFINE_EVENT(net_dev_rx_exit_template, netif_receive_skb_list_exit, · line 345

Quoted 10 lines
include/trace/events/net.h +10 -0
+		__string(	name,		dev->name	)
+		__field(	int,		refcnt		)
+		__field(	void *,		location	)
+	),
+
+	TP_fast_assign(
+		int i, refcnt = 0;
+
+		for_each_possible_cpu(i)
+			refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);

Rather than copying the definition of netdev_refcnt_read here, so just use it.

Quoted 16 lines
include/trace/events/net.h +16 -0
+
+		__assign_str(name, dev->name);
+		__entry->refcnt = refcnt;
+		__entry->location = location;
+	),
+
+	TP_printk("dev=%s refcnt=%d location=%p",
+		__get_str(name), __entry->refcnt, __entry->location)
+);
+
+DEFINE_EVENT(net_dev_refcnt_template, net_dev_put,
+
+	TP_PROTO(struct net_device *dev, void *location),
+
+	TP_ARGS(dev, location)
+);

include/trace/events/net.h DEFINE_EVENT(net_dev_rx_exit_template, netif_receive_skb_list_exit, · line 371

Quoted 10 lines
include/trace/events/net.h +8 -0
+
+DEFINE_EVENT(net_dev_refcnt_template, net_dev_hold,
+
+	TP_PROTO(struct net_device *dev, void *location),
+
+	TP_ARGS(dev, location)
+);
+
 #endif /* _TRACE_NET_H */
 

The location alone does nothing for resolving reference count leaks; you really have to use stack traces to pair up the hold and put and to give context for what are repetitive locations for the hold and put.

#3 lore raw

Re: [PATCH v2 2/2] net: add trace events for net_device refcnt

Quoted 3 lines · David Ahern

The location alone does nothing for resolving reference count leaks; you really have to use stack traces to pair up the hold and put and to give context for what are repetitive locations for the hold and put.

+1

And, as you are using tracepoint, you get trace filter and trigger for free, which already includes stacktrace.