can: rockchip: rk3576: fix rtnl_lock deadlock during interface down under bus traffic
Subsystems (5)
PATCH 1/1 can: rockchip: rk3576: fix rtnl_lock deadlock during interface down under bus traffic
When bringing the CAN interface down (via ip link set can0 down) while there is heavy incoming CAN traffic or continuous hardware error frames, the system hangs and deadlocks. Existing and new networking operations (such as ifconfig, SSH logins, Socket operations) hang indefinitely waiting for rtnl_lock.
The deadlock occurs because rk3576_canfd_close() calls napi_disable() before rk3576_canfd_stop(). Since hardware interrupts are still active, incoming CAN frames and error interrupts continuously trigger napi_schedule(), preventing napi_disable() from seeing the NAPI_STATE_SCHED bit cleared and causing it to loop infinitely in msleep(1). Because dev_close() holds the global rtnl_lock, the entire networking subsystem deadlocks:
Call trace: __switch_to+0xdc/0x120 __schedule+0x2ac/0x840 schedule+0x54/0xe0 schedule_hrtimeout_range_clock+0x98/0x134 usleep_range_state+0x7c/0xb0 napi_disable+0xc0/0x110 rk3576_canfd_close+0x44/0xd0 __dev_close_many+0xb0/0x14c dev_change_flags+0x28/0x64 do_setlink+0x618/0xe2c rtnetlink_rcv_msg+0x2a8/0x380
Fix this by:
1. Reordering rk3576_canfd_close() to call rk3576_canfd_stop() before napi_disable(), ensuring interrupts are disabled and controller is in reset mode before waiting for NAPI to complete.
2. Standardizing rk3576_canfd_rx_poll() to respect the NAPI quota/budget and properly complete NAPI polling via napi_complete_done().
3. Guarding against NULL pointer dereference in rk3576_canfd_err() when alloc_can_err_skb() fails, and calling can_bus_off() upon bus-off.
--- drivers/net/can/rockchip/rk3576_canfd.c | 61 +++++++++++++------------ 1 file changed, 32 insertions(+), 29 deletions(-)
drivers/net/can/rockchip/rk3576_canfd.c +32 -29
@@ -859,31 +859,28 @@ static int rk3576_canfd_rx_poll(struct napi_struct *napi, int quota) {struct net_device *ndev = napi->dev;struct rk3576_canfd *rcan =netdev_priv (ndev); -int work_done =0 , cnt =0 ; +int work_done =0 ; +u32 frames_avail;if (rcan->use_dma) { -while (work_done < rcan->quota) +while (work_done < rcan->quota && work_done < quota) work_done +=rk3576_canfd_rx (ndev, work_done); -if (work_done <= rcan->rx_fifo_depth) { -napi_complete_done (napi, work_done); -rk3576_canfd_write (rcan,CANFD_INT_MASK ,INT_ENABLE ); +if (work_done < quota) { +if (napi_complete_done (napi, work_done)) +rk3576_canfd_write (rcan,CANFD_INT_MASK ,INT_ENABLE ); } }else { - quota = (rk3576_canfd_read (rcan,CANFD_STR_STATE ) & rcan->rx_fifo_mask) >> - rcan->rx_fifo_shift; - quota = quota / rcan->rx_max_data; - cnt = (rk3576_canfd_read (rcan,CANFD_STR_STATE ) &INTM_CNT_MASK ) >>INTM_CNT_SHIFT ; -if (quota != cnt) - quota = ((rk3576_canfd_read (rcan,CANFD_STR_STATE ) & rcan->rx_fifo_mask) >> - rcan->rx_fifo_shift) / rcan->rx_max_data; - -while (work_done < quota) + frames_avail = (rk3576_canfd_read (rcan,CANFD_STR_STATE ) & rcan->rx_fifo_mask) >> + rcan->rx_fifo_shift; + frames_avail = frames_avail / rcan->rx_max_data; + +while (work_done < frames_avail && work_done < quota) work_done +=rk3576_canfd_rx (ndev,CANFD_RXFRD ); -if (work_done <= rcan->rx_fifo_depth) { -napi_complete_done (napi, work_done); -rk3576_canfd_write (rcan,CANFD_INT_MASK ,INT_ENABLE ); +if (work_done < quota) { +if (napi_complete_done (napi, work_done)) +rk3576_canfd_write (rcan,CANFD_INT_MASK ,INT_ENABLE ); } }return work_done;
@@ -926,7 +923,7 @@ static int rk3576_canfd_err(struct net_device *ndev, u32 isr) {struct rk3576_canfd *rcan =netdev_priv (ndev);struct net_device_stats *stats = &ndev->stats; -struct can_frame *cf; +struct can_frame *cf =NULL ;struct sk_buff *skb;unsigned int rxerr, txerr;u32 sta_reg;
@@ -945,17 +942,21 @@ static int rk3576_canfd_err(struct net_device *ndev, u32 isr)if (isr &BUS_OFF_INT ) { rcan->can.state =CAN_STATE_BUS_OFF ; rcan->can.can_stats.bus_off++; - cf->can_id |=CAN_ERR_BUSOFF ; +can_bus_off (ndev); +if (skb) + cf->can_id |=CAN_ERR_BUSOFF ; }else if (isr &PASSIVE_ERR_INT ) { rcan->can.can_stats.error_passive++; rcan->can.state =CAN_STATE_ERROR_PASSIVE ;/* error passive state */ - cf->can_id |=CAN_ERR_CRTL ; - cf->data[1 ] = (txerr > rxerr) ? -CAN_ERR_CRTL_TX_WARNING : -CAN_ERR_CRTL_RX_WARNING ; - cf->data[6 ] = txerr; - cf->data[7 ] = rxerr; +if (skb) { + cf->can_id |=CAN_ERR_CRTL ; + cf->data[1 ] = (txerr > rxerr) ? +CAN_ERR_CRTL_TX_WARNING : +CAN_ERR_CRTL_RX_WARNING ; + cf->data[6 ] = txerr; + cf->data[7 ] = rxerr; + } }if (sta_reg &ERR_WARNING_STATE ) { rcan->can.can_stats.error_warning++;
@@ -984,9 +985,11 @@ static int rk3576_canfd_err(struct net_device *ndev, u32 isr)can_free_echo_skb (ndev,0 ,NULL );netif_start_queue (ndev); } - stats->rx_packets++; - stats->rx_bytes += cf->can_dlc; -netif_rx (skb); +if (skb) { + stats->rx_packets++; + stats->rx_bytes += cf->can_dlc; +netif_rx (skb); + }return 0 ; }
@@ -1070,8 +1073,8 @@ static int rk3576_canfd_close(struct net_device *ndev)struct rk3576_canfd *rcan =netdev_priv (ndev);netif_stop_queue (ndev); -napi_disable (&rcan->napi);rk3576_canfd_stop (ndev); +napi_disable (&rcan->napi);close_candev (ndev);pm_runtime_put (rcan->dev);
Signature · 1 line
2.34.1
Hi Marc, Elaine, and linux-can community,
To help anyone easily reproduce and verify this deadlock scenario on RV1126B / RK3576, below is the full standalone Python SocketCAN test script used to trigger the race condition:
```python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
reproduce_can_deadlock.py
Constructs a concurrent kernel-level deadlock scenario:
1. Blocking sender threads: deep sleep wait in can_send() waiting for TX FIFO.
2. Error frame receiver: setsockopt(CAN_RAW_ERR_FILTER) to receive all error frames.
3. Socket churn: rapid open/bind/send/close.
4. Down/up interrupt loop: calls `ip link set can0 down` during active traffic.
"""
import os
import sys
import time
import socket
import struct
import threading
import subprocess
AF_CAN = getattr(socket, "AF_CAN", 29)
PF_CAN = AF_CAN
CAN_RAW = getattr(socket, "CAN_RAW", 1)
SOL_CAN_BASE = 100
CAN_RAW_FILTER = 1
CAN_RAW_ERR_FILTER = 2
CAN_RAW_LOOPBACK = 3
CAN_RAW_RECV_OWN_MSGS = 4
CAN_RAW_FD_FRAMES = 5
CAN_ERR_MASK = 0x1FFFFFFF
INTERFACE = "can0"
STOP_EVENT = threading.Event()
DEADLOCK_DETECTED = threading.Event()
def build_can_frame(can_id, data):
can_dlc = len(data)
data = data.ljust(8, b'\x00')
return struct.pack("=IB3x8s", can_id, can_dlc, data)
def build_canfd_frame(can_id, data, flags=0):
length = len(data)
data = data.ljust(64, b'\x00')
return struct.pack("=IBB2x64s", can_id, length, flags, data)
def sender_thread(thread_id, is_fd=False):
sock = socket.socket(PF_CAN, socket.SOCK_RAW, CAN_RAW)
try:
if is_fd:
try:
sock.setsockopt(SOL_CAN_BASE, CAN_RAW_FD_FRAMES, 1)
except Exception:
pass
sock.bind((INTERFACE,))
sock.setblocking(True)
except Exception:
sock.close()
return
frame_data = b'\x11\x22\x33\x44\x55\x66\x77\x88'
can_id = 0x100 + thread_id
frame = build_can_frame(can_id, frame_data)
while not STOP_EVENT.is_set():
try:
sock.send(frame)
except socket.error:
time.sleep(0.001)
except Exception:
break
sock.close()
def receiver_thread():
try:
sock = socket.socket(PF_CAN, socket.SOCK_RAW, CAN_RAW)
sock.setsockopt(SOL_CAN_BASE, CAN_RAW_ERR_FILTER, CAN_ERR_MASK)
sock.bind((INTERFACE,))
sock.settimeout(0.1)
except Exception:
return
while not STOP_EVENT.is_set():
try:
_ = sock.recv(72)
except socket.timeout:
continue
except Exception:
time.sleep(0.01)
sock.close()
def socket_churn_thread():
while not STOP_EVENT.is_set():
try:
s = socket.socket(PF_CAN, socket.SOCK_RAW, CAN_RAW)
s.bind((INTERFACE,))
s.setblocking(False)
frame = build_can_frame(0x200, b'\xaa\xbb\xcc\xdd')
try:
s.send(frame)
except Exception:
pass
s.close()
except Exception:
pass
time.sleep(0.002)
def main():
if os.geteuid() != 0:
print("[-] Please run as root: sudo python3 reproduce_can_deadlock.py")
sys.exit(1)
print("=" * 65)
print("Starting SocketCAN deadlock reproduction tool")
print("=" * 65)
subprocess.run(["ip", "link", "set", INTERFACE, "down"], stderr=subprocess.DEVNULL)
time.sleep(0.3)
subprocess.run(["ip", "link", "set", INTERFACE, "txqueuelen", "1"], check=False)
subprocess.run([
"ip", "link", "set", INTERFACE, "up", "type", "can",
"bitrate", "500000", "dbitrate", "500000", "fd", "on", "restart-ms", "100"
], check=False)
threads = []
for i in range(3):
t = threading.Thread(target=sender_thread, args=(i, i % 2 == 0), daemon=True)
t.start()
threads.append(t)
t_recv = threading.Thread(target=receiver_thread, daemon=True)
t_recv.start()
threads.append(t_recv)
t_churn = threading.Thread(target=socket_churn_thread, daemon=True)
t_churn.start()
threads.append(t_churn)
loop_count = 0
try:
while True:
loop_count += 1
now_str = time.strftime("%H:%M:%S")
sys.stdout.write(f"[{now_str}] Loop {loop_count:5d}: Bringing can0 down... ")
sys.stdout.flush()
t0 = time.time()
p_down = subprocess.Popen(["ip", "link", "set", INTERFACE, "down"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
p_down.communicate(timeout=4.0)
down_cost = time.time() - t0
sys.stdout.write(f"Done ({down_cost*1000:.1f}ms) -> Bringing can0 up... ")
sys.stdout.flush()
except subprocess.TimeoutExpired:
print("\n" + "!" * 65)
print("Captured deadlock! `ip link set can0 down` hung > 4s!")
print("Kernel rtnl_lock is deadlocked!")
print("!" * 65)
DEADLOCK_DETECTED.set()
break
t1 = time.time()
p_up = subprocess.Popen(["ip", "link", "set", INTERFACE, "up"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
p_up.communicate(timeout=4.0)
up_cost = time.time() - t1
sys.stdout.write(f"Done ({up_cost*1000:.1f}ms)\n")
sys.stdout.flush()
except subprocess.TimeoutExpired:
print("\n" + "!" * 65)
print("Captured deadlock! `ip link set can0 up` hung > 4s!")
print("!" * 65)
DEADLOCK_DETECTED.set()
break
time.sleep(0.005)
except KeyboardInterrupt:
print("\n[*] Exiting test...")
finally:
STOP_EVENT.set()
if not DEADLOCK_DETECTED.is_set():
subprocess.run(["ip", "link", "set", INTERFACE, "down"], stderr=subprocess.DEVNULL)
print(f"[*] Completed {loop_count} loops.")
if __name__ == "__main__":
main()
```Also, if maintainers prefer this fix to be split into separate smaller atomic patches (e.g. 1. close deadlock fix, 2. NAPI budget handling, 3. error handling NULL check), please let me know and I will gladly send a v2 patch series!
Best regards,
Cheng Liu
Marc Kleine-Budde
maintainerQuoted 37 lines · Cheng Liu
When bringing the CAN interface down (via
ip link set can0 down) while there is heavy incoming CAN traffic or continuous hardware error frames, the system hangs and deadlocks. Existing and new networking operations (such asifconfig, SSH logins, Socket operations) hang indefinitely waiting forrtnl_lock.The deadlock occurs because
rk3576_canfd_close()callsnapi_disable()beforerk3576_canfd_stop(). Since hardware interrupts are still active, incoming CAN frames and error interrupts continuously triggernapi_schedule(), preventingnapi_disable()from seeing theNAPI_STATE_SCHEDbit cleared and causing it to loop infinitely inmsleep(1). Becausedev_close()holds the globalrtnl_lock, the entire networking subsystem deadlocks:Call trace: __switch_to+0xdc/0x120 __schedule+0x2ac/0x840 schedule+0x54/0xe0 schedule_hrtimeout_range_clock+0x98/0x134 usleep_range_state+0x7c/0xb0 napi_disable+0xc0/0x110 rk3576_canfd_close+0x44/0xd0 __dev_close_many+0xb0/0x14c dev_change_flags+0x28/0x64 do_setlink+0x618/0xe2c rtnetlink_rcv_msg+0x2a8/0x380Fix this by:
1. Reorderingrk3576_canfd_close()to callrk3576_canfd_stop()beforenapi_disable(), ensuring interrupts are disabled and controller is in reset mode before waiting for NAPI to complete.
2. Standardizingrk3576_canfd_rx_poll()to respect the NAPI quota/budget and properly complete NAPI polling vianapi_complete_done().
3. Guarding against NULL pointer dereference inrk3576_canfd_err()whenalloc_can_err_skb()fails, and callingcan_bus_off()upon bus-off.
Quoted 3 lines
Signed-off-by Cheng Liu <[email protected]>--- drivers/net/can/rockchip/rk3576_canfd.c | 61 +++++++++++++------------
The driver for the rk3576 is not mainline, yet. Maybe Elaine can pick it up.
regards,
Marc
Signature · 4 lines
Pengutronix e.K. | Marc Kleine-Budde | Embedded Linux | https://www.pengutronix.de | Vertretung Nürnberg | Phone: +49-5121-206917-129 | Amtsgericht Hildesheim, HRA 2686 | Fax: +49-5121-206917-9 |