virtio-net: disable delayed refill when pausing rx
Subsystems (6)
v3 cover virtio-net: disable delayed refill when pausing rx
Hi everyone,
This series tries to fix a deadlock in virtio-net when binding/unbinding XDP program, XDP socket or resizing the rx queue.
When pausing rx (e.g. set up xdp, xsk pool, rx resize), we call napi_disable() on the receive queue's napi. In delayed refill_work, it also calls napi_disable() on the receive queue's napi. When napi_disable() is called on an already disabled napi, it will sleep in napi_disable_locked while still holding the netdev_lock. As a result, later napi_enable gets stuck too as it cannot acquire the netdev_lock. This leads to refill_work and the pause-then-resume tx are stuck altogether.
This scenario can be reproducible by binding a XDP socket to virtio-net interface without setting up the fill ring. As a result, try_fill_recv will fail until the fill ring is set up and refill_work is scheduled.
This fix adds virtnet_rx_(pause/resume)_all helpers and fixes up the virtnet_rx_resume to disable future and cancel all inflights delayed refill_work before calling napi_disable() to pause the rx.
Version 3 changes:
- Patch 1: refactor to avoid code duplication
Version 2 changes:
- Add selftest for deadlock scenario
Thanks,
Quang Minh.
Bui Quang Minh (3):
virtio-net: disable delayed refill when pausing rx
selftests: net: move xdp_helper to net/lib
selftests: net: add a virtio_net deadlock selftest
drivers/net/virtio_net.c | 69 +++++++++++++++----
tools/testing/selftests/Makefile | 2 +-
tools/testing/selftests/drivers/net/Makefile | 2 -
tools/testing/selftests/drivers/net/queues.py | 4 +-
.../selftests/drivers/net/virtio_net/Makefile | 2 +
.../selftests/drivers/net/virtio_net/config | 1 +
.../drivers/net/virtio_net/lib/py/__init__.py | 16 +++++
.../drivers/net/virtio_net/xsk_pool.py | 52 ++++++++++++++
tools/testing/selftests/net/lib/.gitignore | 1 +
tools/testing/selftests/net/lib/Makefile | 1 +
.../{drivers/net => net/lib}/xdp_helper.c | 0
11 files changed, 133 insertions(+), 17 deletions(-)
create mode 100644 tools/testing/selftests/drivers/net/virtio_net/lib/py/__init__.py
create mode 100755 tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py
rename tools/testing/selftests/{drivers/net => net/lib}/xdp_helper.c (100%)Signature · 1 line
2.43.0
PATCH v3 1/3 virtio-net: disable delayed refill when pausing rx
When pausing rx (e.g. set up xdp, xsk pool, rx resize), we call napi_disable() on the receive queue's napi. In delayed refill_work, it also calls napi_disable() on the receive queue's napi. When napi_disable() is called on an already disabled napi, it will sleep in napi_disable_locked while still holding the netdev_lock. As a result, later napi_enable gets stuck too as it cannot acquire the netdev_lock. This leads to refill_work and the pause-then-resume tx are stuck altogether.
This scenario can be reproducible by binding a XDP socket to virtio-net interface without setting up the fill ring. As a result, try_fill_recv will fail until the fill ring is set up and refill_work is scheduled.
This commit adds virtnet_rx_(pause/resume)_all helpers and fixes up the virtnet_rx_resume to disable future and cancel all inflights delayed refill_work before calling napi_disable() to pause the rx.
--- drivers/net/virtio_net.c | 69 +++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 12 deletions(-)
drivers/net/virtio_net.c +57 -12
@@ -3342,7 +3342,8 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)return NETDEV_TX_OK ; } -static void virtnet_rx_pause (struct virtnet_info *vi,struct receive_queue *rq) +static void __virtnet_rx_pause (struct virtnet_info *vi, +struct receive_queue *rq) {bool running =netif_running (vi->dev);
@@ -3352,17 +3353,63 @@ static void virtnet_rx_pause(struct virtnet_info *vi, struct receive_queue *rq) } } -static void virtnet_rx_resume (struct virtnet_info *vi,struct receive_queue *rq) +static void virtnet_rx_pause_all (struct virtnet_info *vi) +{ +int i; + +/* +* Make sure refill_work does not run concurrently to +* avoid napi_disable race which leads to deadlock. +*/ +disable_delayed_refill (vi); +cancel_delayed_work_sync (&vi->refill); +for (i =0 ; i < vi->max_queue_pairs; i++) +__virtnet_rx_pause (vi, &vi->rq[i]); +} + +static void virtnet_rx_pause (struct virtnet_info *vi,struct receive_queue *rq) +{ +/* +* Make sure refill_work does not run concurrently to +* avoid napi_disable race which leads to deadlock. +*/ +disable_delayed_refill (vi); +cancel_delayed_work_sync (&vi->refill); +__virtnet_rx_pause (vi, rq); +} + +static void __virtnet_rx_resume (struct virtnet_info *vi, +struct receive_queue *rq, +bool refill) {bool running =netif_running (vi->dev); -if (!try_fill_recv (vi, rq,GFP_KERNEL )) +if (refill && !try_fill_recv (vi, rq,GFP_KERNEL ))schedule_delayed_work (&vi->refill,0 );if (running)virtnet_napi_enable (rq); } +static void virtnet_rx_resume_all (struct virtnet_info *vi) +{ +int i; + +enable_delayed_refill (vi); +for (i =0 ; i < vi->max_queue_pairs; i++) { +if (i < vi->curr_queue_pairs) +__virtnet_rx_resume (vi, &vi->rq[i], true); +else +__virtnet_rx_resume (vi, &vi->rq[i], false); + } +} + +static void virtnet_rx_resume (struct virtnet_info *vi,struct receive_queue *rq) +{ +enable_delayed_refill (vi); +__virtnet_rx_resume (vi, rq, true); +} +static int virtnet_rx_resize (struct virtnet_info *vi,struct receive_queue *rq,u32 ring_num) {
@@ -5959,12 +6006,12 @@ static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,if (prog)bpf_prog_add (prog, vi->max_queue_pairs -1 ); +virtnet_rx_pause_all (vi); +/* Make sure NAPI is not using any XDP TX queues for RX. */ if (netif_running (dev)) { -for (i =0 ; i < vi->max_queue_pairs; i++) { -virtnet_napi_disable (&vi->rq[i]); +for (i =0 ; i < vi->max_queue_pairs; i++)virtnet_napi_tx_disable (&vi->sq[i]); - } }if (!prog) {
@@ -5996,13 +6043,12 @@ static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog, vi->xdp_enabled = false; } +virtnet_rx_resume_all (vi);for (i =0 ; i < vi->max_queue_pairs; i++) {if (old_prog)bpf_prog_put (old_prog); -if (netif_running (dev)) { -virtnet_napi_enable (&vi->rq[i]); +if (netif_running (dev))virtnet_napi_tx_enable (&vi->sq[i]); - } }return 0 ;
@@ -6014,11 +6060,10 @@ static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,rcu_assign_pointer (vi->rq[i].xdp_prog, old_prog); } +virtnet_rx_resume_all (vi);if (netif_running (dev)) { -for (i =0 ; i < vi->max_queue_pairs; i++) { -virtnet_napi_enable (&vi->rq[i]); +for (i =0 ; i < vi->max_queue_pairs; i++)virtnet_napi_tx_enable (&vi->sq[i]); - } }if (prog)bpf_prog_sub (prog, vi->max_queue_pairs -1 );
Signature · 1 line
2.43.0
Quoted 13 lines · Bui Quang Minh
When pausing rx (e.g. set up xdp, xsk pool, rx resize), we call
napi_disable()on the receive queue's napi. In delayedrefill_work, it also callsnapi_disable()on the receive queue's napi. Whennapi_disable()is called on an already disabled napi, it will sleep innapi_disable_lockedwhile still holding thenetdev_lock. As a result, laternapi_enablegets stuck too as it cannot acquire thenetdev_lock. This leads torefill_workand the pause-then-resume tx are stuck altogether.This scenario can be reproducible by binding a XDP socket to virtio-net interface without setting up the fill ring. As a result,
try_fill_recvwill fail until the fill ring is set up andrefill_workis scheduled.
Quoted 6 lines
This commit adds virtnet_rx_(pause/resume)_all helpers and fixes up the
virtnet_rx_resumeto disable future and cancel all inflights delayedrefill_workbefore callingnapi_disable()to pause the rx.Fixes 413f0271f396 ("net: protect NAPI enablement with netdev_lock()")Signed-off-by Bui Quang Minh <[email protected]>
Quoted 132 lines
--- drivers/net/virtio_net.c | 69 +++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 12 deletions(-)drivers/net/virtio_net.c +57
-12@@ -3342,7 +3342,8 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)return NETDEV_TX_OK ; } -static void virtnet_rx_pause (struct virtnet_info *vi,struct receive_queue *rq) +static void __virtnet_rx_pause (struct virtnet_info *vi, +struct receive_queue *rq) {bool running =netif_running (vi->dev);@@ -3352,17 +3353,63 @@ static void virtnet_rx_pause(struct virtnet_info *vi, struct receive_queue *rq) } } -static void virtnet_rx_resume (struct virtnet_info *vi,struct receive_queue *rq) +static void virtnet_rx_pause_all (struct virtnet_info *vi) +{ +int i; + +/* +* Make sure refill_work does not run concurrently to +* avoid napi_disable race which leads to deadlock. +*/ +disable_delayed_refill (vi); +cancel_delayed_work_sync (&vi->refill); +for (i =0 ; i < vi->max_queue_pairs; i++) +__virtnet_rx_pause (vi, &vi->rq[i]); +} + +static void virtnet_rx_pause (struct virtnet_info *vi,struct receive_queue *rq) +{ +/* +* Make sure refill_work does not run concurrently to +* avoid napi_disable race which leads to deadlock. +*/ +disable_delayed_refill (vi); +cancel_delayed_work_sync (&vi->refill); +__virtnet_rx_pause (vi, rq); +} + +static void __virtnet_rx_resume (struct virtnet_info *vi, +struct receive_queue *rq, +bool refill) {bool running =netif_running (vi->dev); -if (!try_fill_recv (vi, rq,GFP_KERNEL )) +if (refill && !try_fill_recv (vi, rq,GFP_KERNEL ))schedule_delayed_work (&vi->refill,0 );if (running)virtnet_napi_enable (rq); } +static void virtnet_rx_resume_all (struct virtnet_info *vi) +{ +int i; + +enable_delayed_refill (vi); +for (i =0 ; i < vi->max_queue_pairs; i++) { +if (i < vi->curr_queue_pairs) +__virtnet_rx_resume (vi, &vi->rq[i], true); +else +__virtnet_rx_resume (vi, &vi->rq[i], false); + } +} + +static void virtnet_rx_resume (struct virtnet_info *vi,struct receive_queue *rq) +{ +enable_delayed_refill (vi); +__virtnet_rx_resume (vi, rq, true); +} +static int virtnet_rx_resize (struct virtnet_info *vi,struct receive_queue *rq,u32 ring_num) {@@ -5959,12 +6006,12 @@ static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,if (prog)bpf_prog_add (prog, vi->max_queue_pairs -1 ); +virtnet_rx_pause_all (vi); +/* Make sure NAPI is not using any XDP TX queues for RX. */ if (netif_running (dev)) { -for (i =0 ; i < vi->max_queue_pairs; i++) { -virtnet_napi_disable (&vi->rq[i]); +for (i =0 ; i < vi->max_queue_pairs; i++)virtnet_napi_tx_disable (&vi->sq[i]); - } }if (!prog) {@@ -5996,13 +6043,12 @@ static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog, vi->xdp_enabled = false; } +virtnet_rx_resume_all (vi);for (i =0 ; i < vi->max_queue_pairs; i++) {if (old_prog)bpf_prog_put (old_prog); -if (netif_running (dev)) { -virtnet_napi_enable (&vi->rq[i]); +if (netif_running (dev))virtnet_napi_tx_enable (&vi->sq[i]); - } }return 0 ;@@ -6014,11 +6060,10 @@ static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,rcu_assign_pointer (vi->rq[i].xdp_prog, old_prog); } +virtnet_rx_resume_all (vi);if (netif_running (dev)) { -for (i =0 ; i < vi->max_queue_pairs; i++) { -virtnet_napi_enable (&vi->rq[i]); +for (i =0 ; i < vi->max_queue_pairs; i++)virtnet_napi_tx_enable (&vi->sq[i]); - } }if (prog)bpf_prog_sub (prog, vi->max_queue_pairs -1 );Signature · 1 line
2.43.0
PATCH v3 2/3 selftests: net: move xdp_helper to net/lib
In reply to Bui Quang MinhMove xdp_helper to net/lib to make it easier for other selftests to use the helper.
---
tools/testing/selftests/drivers/net/Makefile | 2 --
tools/testing/selftests/drivers/net/queues.py | 4 ++--
tools/testing/selftests/net/lib/.gitignore | 1 +
tools/testing/selftests/net/lib/Makefile | 1 +
tools/testing/selftests/{drivers/net => net/lib}/xdp_helper.c | 0
5 files changed, 4 insertions(+), 4 deletions(-)
rename tools/testing/selftests/{drivers/net => net/lib}/xdp_helper.c (100%)tools/testing/selftests/drivers/net/Makefile +0 -2
@@ -6,8 +6,6 @@ TEST_INCLUDES := $(wildcard lib/py/*.py) \ ../../net/net_helper.sh \ ../../net/lib.sh \ -TEST_GEN_FILES := xdp_helper - TEST_PROGS := \ netcons_basic.sh \ netcons_fragmented_msg.sh \
tools/testing/selftests/drivers/net/queues.py +2 -2
@@ -26,13 +26,13 @@ def nl_get_queues(cfg, nl, qtype='rx'): def check_xsk(cfg, nl, xdp_queue_id=0) -> None: # Probe for support - xdp = cmd(f'{cfg.test_dir / "xdp_helper"} - -', fail=False) + xdp = cmd(f'{cfg.net_lib_dir / "xdp_helper"} - -', fail=False) if xdp.ret == 255: raise KsftSkipEx('AF_XDP unsupported') elif xdp.ret > 0: raise KsftFailEx('unable to create AF_XDP socket') - with bkg(f'{cfg.test_dir / "xdp_helper"} {cfg.ifindex} {xdp_queue_id}', + with bkg(f'{cfg.net_lib_dir / "xdp_helper"} {cfg.ifindex} {xdp_queue_id}', ksft_wait=3): rx = tx = False
tools/testing/selftests/net/lib/.gitignore +1 -0
@@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only csum +xdp_helper
tools/testing/selftests/net/lib/Makefile +1 -0
@@ -10,6 +10,7 @@ TEST_FILES += ../../../../net/ynl TEST_GEN_FILES += csum TEST_GEN_FILES += $(patsubst %.c,%.o,$(wildcard *.bpf.c)) +TEST_GEN_FILES += xdp_helper TEST_INCLUDES := $(wildcard py/*.py sh/*.sh)
tools/testing/selftests/net/lib/xdp_helper.c renamed from tools/testing/selftests/drivers/net/xdp_helper.c +0 -0
Signature · 1 line
2.43.0
PATCH v3 3/3 selftests: net: add a virtio_net deadlock selftest
In reply to Bui Quang MinhThe selftest reproduces the deadlock scenario when binding/unbinding XDP program, XDP socket, rx ring resize on virtio_net interface.
--- tools/testing/selftests/Makefile | 2 +- .../selftests/drivers/net/virtio_net/Makefile | 2 + .../selftests/drivers/net/virtio_net/config | 1 + .../drivers/net/virtio_net/lib/py/__init__.py | 16 ++++++ .../drivers/net/virtio_net/xsk_pool.py | 52 +++++++++++++++++++ 5 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/drivers/net/virtio_net/lib/py/__init__.py create mode 100755 tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py
tools/testing/selftests/Makefile +1 -1
@@ -130,7 +130,7 @@ TARGETS_HOTPLUG = cpu-hotplug TARGETS_HOTPLUG += memory-hotplug # Networking tests want the net/lib target, include it automatically -ifneq ($(filter net drivers/net drivers/net/hw,$(TARGETS)),) +ifneq ($(filter net drivers/net drivers/net/hw drivers/net/virtio_net,$(TARGETS)),) ifeq ($(filter net/lib,$(TARGETS)),) INSTALL_DEP_TARGETS := net/lib endif
tools/testing/selftests/drivers/net/virtio_net/Makefile +2 -0
@@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0+ OR MIT TEST_PROGS = basic_features.sh \ + xsk_pool.py \ # TEST_FILES = \
@@ -8,6 +9,7 @@ TEST_FILES = \ # TEST_INCLUDES = \ + $(wildcard lib/py/*.py ../lib/py/*.py) \ ../../../net/forwarding/lib.sh \ ../../../net/lib.sh \ #
tools/testing/selftests/drivers/net/virtio_net/config +1 -0
@@ -6,3 +6,4 @@ CONFIG_NET_L3_MASTER_DEV=y CONFIG_NET_VRF=m CONFIG_VIRTIO_DEBUG=y CONFIG_VIRTIO_NET=y +CONFIG_XDP_SOCKETS=y
tools/testing/selftests/drivers/net/virtio_net/lib/py/__init__.py new file +16 -0
@@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0 + +import sys +from pathlib import Path + +KSFT_DIR = (Path(__file__).parent / "../../../../..").resolve() + +try: + sys.path.append(KSFT_DIR.as_posix()) + from net.lib.py import * + from drivers.net.lib.py import * +except ModuleNotFoundError as e: + ksft_pr("Failed importing `net` library from kernel sources") + ksft_pr(str(e)) + ktap_result(True, comment="SKIP") + sys.exit(4)
tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py new file +52 -0
52 lines not on this page · raw message
Signature · 1 line
2.43.0
Jakub Kicinski
maintainerRe: [PATCH v3 3/3] selftests: net: add a virtio_net deadlock selftest
Quoted 10 lines · On Tue, 15 Apr 2025 14:43:41 +0700 Bui Quang Minh wrote:
tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py new file +10
-0+def setup_xsk(cfg, xdp_queue_id = 0) -> bkg: + # Probe for support + xdp = cmd(f'{cfg.net_lib_dir / "xdp_helper"} - -', fail=False) + if xdp.ret == 255: + raise KsftSkipEx('AF_XDP unsupported') + elif xdp.ret > 0: + raise KsftFailEx('unable to create AF_XDP socket') + + return bkg(f'{cfg.net_lib_dir / "xdp_helper"} {cfg.ifindex} {xdp_queue_id}', + ksft_wait=3)
tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py line 41
Quoted 10 lines
tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py new file +10
-0+ +def check_xdp_bind(cfg): + ip(f"link set dev %s xdp obj %s sec xdp" % + (cfg.ifname, cfg.net_lib_dir / "xdp_dummy.bpf.o")) + ip(f"link set dev %s xdp off" % cfg.ifname) + +def check_rx_resize(cfg, queue_size = 128): + rx_ring = _get_rx_ring_entries(cfg) + ethtool(f"-G %s rx %d" % (cfg.ifname, queue_size)) + ethtool(f"-G %s rx %d" % (cfg.ifname, rx_ring))
Unfortunately this doesn't work on a basic QEMU setup:
# ethtool -G eth0 rx 128 [ 15.680655][ T287] virtio_net virtio2 eth0: resize rx fail: rx queue index: 0 err: -2 netlink error: No such file or directory
Is there a way to enable more capable virtio_net with QEMU?
Re: [PATCH v3 3/3] selftests: net: add a virtio_net deadlock selftest
Quoted 23 lines · Jakub Kicinski
Quoted 10 lines · On Tue, 15 Apr 2025 14:43:41 +0700 Bui Quang Minh wrote:
tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py new file +10
-0+def setup_xsk(cfg, xdp_queue_id = 0) -> bkg: + # Probe for support + xdp = cmd(f'{cfg.net_lib_dir / "xdp_helper"} - -', fail=False) + if xdp.ret == 255: + raise KsftSkipEx('AF_XDP unsupported') + elif xdp.ret > 0: + raise KsftFailEx('unable to create AF_XDP socket') + + return bkg(f'{cfg.net_lib_dir / "xdp_helper"} {cfg.ifindex} {xdp_queue_id}', + ksft_wait=3)tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py line 41
Quoted 10 lines
tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py new file +10
-0+ +def check_xdp_bind(cfg): + ip(f"link set dev %s xdp obj %s sec xdp" % + (cfg.ifname, cfg.net_lib_dir / "xdp_dummy.bpf.o")) + ip(f"link set dev %s xdp off" % cfg.ifname) + +def check_rx_resize(cfg, queue_size = 128): + rx_ring = _get_rx_ring_entries(cfg) + ethtool(f"-G %s rx %d" % (cfg.ifname, queue_size)) + ethtool(f"-G %s rx %d" % (cfg.ifname, rx_ring))Unfortunately this doesn't work on a basic QEMU setup:
Quoted 5 lines
# ethtool -G eth0 rx 128 [ 15.680655][ T287] virtio_net virtio2 eth0: resize rx fail: rx queue index: 0 err: -2 netlink error: No such file or directoryIs there a way to enable more capable
virtio_netwith QEMU?
I guess that virtio-pci-legacy is used in your setup.
Here is how I setup virtio-net with Qemu
-netdev tap,id=hostnet1,vhost=on,script=$NETWORK_SCRIPT,downscript=no \
-device virtio-net-pci,netdev=hostnet1,iommu_platform=on,disable-legacy=on \
The iommu_platform=on is necessary to make vring use dma API which is a requirement to enable xsk_pool in virtio-net (XDP socket will be in zerocopy mode for this case). Otherwise, the XDP socket will fallback to copy mode, xsk_pool is not enabled in virtio-net that makes the probability to reproduce bug to be very small. Currently, when you don't have iommu_platform=on, you can pass the test even before the fix, so I think I will try to harden the selftest to make it return skip in this case.
Thanks,
Quang Minh.
Re: [PATCH v3 3/3] selftests: net: add a virtio_net deadlock selftest
Quoted 25 lines · Bui Quang Minh
Quoted 23 lines · Jakub Kicinski
Quoted 10 lines · On Tue, 15 Apr 2025 14:43:41 +0700 Bui Quang Minh wrote:
tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py new file +10
-0+def setup_xsk(cfg, xdp_queue_id = 0) -> bkg: + # Probe for support + xdp = cmd(f'{cfg.net_lib_dir / "xdp_helper"} - -', fail=False) + if xdp.ret == 255: + raise KsftSkipEx('AF_XDP unsupported') + elif xdp.ret > 0: + raise KsftFailEx('unable to create AF_XDP socket') + + return bkg(f'{cfg.net_lib_dir / "xdp_helper"} {cfg.ifindex} {xdp_queue_id}', + ksft_wait=3)tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py line 41
Quoted 10 lines
tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py new file +10
-0+ +def check_xdp_bind(cfg): + ip(f"link set dev %s xdp obj %s sec xdp" % + (cfg.ifname, cfg.net_lib_dir / "xdp_dummy.bpf.o")) + ip(f"link set dev %s xdp off" % cfg.ifname) + +def check_rx_resize(cfg, queue_size = 128): + rx_ring = _get_rx_ring_entries(cfg) + ethtool(f"-G %s rx %d" % (cfg.ifname, queue_size)) + ethtool(f"-G %s rx %d" % (cfg.ifname, rx_ring))Unfortunately this doesn't work on a basic QEMU setup:
Quoted 5 lines
Quoted 5 lines
# ethtool -G eth0 rx 128 [ 15.680655][ T287] virtio_net virtio2 eth0: resize rx fail: rx queue index: 0 err: -2 netlink error: No such file or directoryIs there a way to enable more capable
virtio_netwith QEMU?
What's the qemu command line and version?
Resize depends on queue_reset which should be supported from Qemu 7.2
Quoted 2 lines
I guess that virtio-pci-legacy is used in your setup.
Note that modern devices are used by default.
Quoted 7 lines
Here is how I setup virtio-net with Qemu
-netdev tap,id=hostnet1,vhost=on,script=$NETWORK_SCRIPT,downscript=no \ -devicevirtio-net-pci,netdev=hostnet1,iommu_platform=on,disable-legacy=on \
Quoted 7 lines
The iommu_platform=on is necessary to make vring use dma API which is a requirement to enable
xsk_poolin virtio-net (XDP socket will be in zerocopy mode for this case). Otherwise, the XDP socket will fallback to copy mode,xsk_poolis not enabled in virtio-net that makes the probability to reproduce bug to be very small. Currently, when you don't have iommu_platform=on, you can pass the test even before the fix, so I think I will try to harden the selftest to make it return skip in this case.
I would like to keep the resize test as it doesn't require iommu_platform.
Thanks
Quoted 4 lines
Thanks,
Quang Minh.
Re: [PATCH v3 3/3] selftests: net: add a virtio_net deadlock selftest
Quoted 43 lines · Jason Wang
Quoted 24 lines · Bui Quang Minh
Quoted 23 lines · Jakub Kicinski
Quoted 10 lines · On Tue, 15 Apr 2025 14:43:41 +0700 Bui Quang Minh wrote:
tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py new file +10
-0+def setup_xsk(cfg, xdp_queue_id = 0) -> bkg: + # Probe for support + xdp = cmd(f'{cfg.net_lib_dir / "xdp_helper"} - -', fail=False) + if xdp.ret == 255: + raise KsftSkipEx('AF_XDP unsupported') + elif xdp.ret > 0: + raise KsftFailEx('unable to create AF_XDP socket') + + return bkg(f'{cfg.net_lib_dir / "xdp_helper"} {cfg.ifindex} {xdp_queue_id}', + ksft_wait=3)tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py line 41
Quoted 10 lines
tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py new file +10
-0+ +def check_xdp_bind(cfg): + ip(f"link set dev %s xdp obj %s sec xdp" % + (cfg.ifname, cfg.net_lib_dir / "xdp_dummy.bpf.o")) + ip(f"link set dev %s xdp off" % cfg.ifname) + +def check_rx_resize(cfg, queue_size = 128): + rx_ring = _get_rx_ring_entries(cfg) + ethtool(f"-G %s rx %d" % (cfg.ifname, queue_size)) + ethtool(f"-G %s rx %d" % (cfg.ifname, rx_ring))Unfortunately this doesn't work on a basic QEMU setup:
Quoted 5 lines
Quoted 5 lines
# ethtool -G eth0 rx 128 [ 15.680655][ T287] virtio_net virtio2 eth0: resize rx fail: rx queue index: 0 err: -2 netlink error: No such file or directoryIs there a way to enable more capable
virtio_netwith QEMU?What's the qemu command line and version?
Resize depends on
queue_resetwhich should be supported from Qemu 7.2Quoted 1 line
I guess that virtio-pci-legacy is used in your setup.
Note that modern devices are used by default.
Quoted 6 lines
Here is how I setup virtio-net with Qemu
-netdev tap,id=hostnet1,vhost=on,script=$NETWORK_SCRIPT,downscript=no \ -devicevirtio-net-pci,netdev=hostnet1,iommu_platform=on,disable-legacy=on \
Quoted 8 lines
Quoted 7 lines
The iommu_platform=on is necessary to make vring use dma API which is a requirement to enable
xsk_poolin virtio-net (XDP socket will be in zerocopy mode for this case). Otherwise, the XDP socket will fallback to copy mode,xsk_poolis not enabled in virtio-net that makes the probability to reproduce bug to be very small. Currently, when you don't have iommu_platform=on, you can pass the test even before the fix, so I think I will try to harden the selftest to make it return skip in this case.I would like to keep the resize test as it doesn't require
iommu_platform.
Okay, in next version I will force the XDP socket binding to zerocopy to setup xsk_pool. When the binding fails, 2 tests still run but I will print a warning message.
Thanks,
Quang Minh.
Jakub Kicinski
maintainerRe: [PATCH v3 3/3] selftests: net: add a virtio_net deadlock selftest
In reply to Jason WangQuoted 8 lines · On Wed, 16 Apr 2025 15:46:42 +0800 Jason Wang wrote:
Quoted 7 lines · Bui Quang Minh
Quoted 6 lines · Jakub Kicinski
Unfortunately this doesn't work on a basic QEMU setup:
# ethtool -G eth0 rx 128 [ 15.680655][ T287] virtio_net virtio2 eth0: resize rx fail: rx queue index: 0 err: -2 netlink error: No such file or directory
Quoted 5 lines
Quoted 1 line
Quoted 1 line
Is there a way to enable more capable
virtio_netwith QEMU?What's the qemu command line and version?
Resize depends on
queue_resetwhich should be supported from Qemu 7.2
I'm using virtme-ng with --net loop and:
QEMU emulator version 9.1.3 (qemu-9.1.3-2.fc41)
--net loop resolves to:
-device virtio-net-device,netdev=n0 \ -netdev hubport,id=n0,hubid=0 \ -device virtio-net-device,netdev=n1 \ -netdev hubport,id=n1,hubid=0
Quoted 10 lines
Quoted 1 line
I guess that virtio-pci-legacy is used in your setup.
Note that modern devices are used by default.
Quoted 6 lines
Here is how I setup virtio-net with Qemu
-netdev tap,id=hostnet1,vhost=on,script=$NETWORK_SCRIPT,downscript=no \ -devicevirtio-net-pci,netdev=hostnet1,iommu_platform=on,disable-legacy=on \
That works! I rejigged the CI, for posterity I used two times:
-device virtio-net-pci,netdev=n0,iommu_platform=on,disable-legacy=on,mq=on,vectors=18
-netdev tap,id=n0,ifname=tap4,vhost=on,script=no,downscript=no,queues=8
and then manually bridged the taps together on the hypervisor side.
Quoted 9 lines
Quoted 7 lines
The iommu_platform=on is necessary to make vring use dma API which is a requirement to enable
xsk_poolin virtio-net (XDP socket will be in zerocopy mode for this case). Otherwise, the XDP socket will fallback to copy mode,xsk_poolis not enabled in virtio-net that makes the probability to reproduce bug to be very small. Currently, when you don't have iommu_platform=on, you can pass the test even before the fix, so I think I will try to harden the selftest to make it return skip in this case.I would like to keep the resize test as it doesn't require
iommu_platform.
Sounds good but lets just add them to the drivers/net/hw directory. I don't think there's anything virtio specific in the test itself?
Right now drivers/net/virtio_net has a test which expects to see both netdevs in the VM, while drivers / Python based tests expect to have the env prepared where only one end is on the local machine, and the other is accessible over SSH or in another netns. So it's a bit painful to marry the two kinds of tests in the CI. At least our netdev CI does not know how to figure this out :( It preps the env and then runs the whole kselftest TARGET in the same setup.
Quoted 22 lines · Bui Quang Minh
Hi everyone,
This series tries to fix a deadlock in virtio-net when binding/unbinding XDP program, XDP socket or resizing the rx queue.
When pausing rx (e.g. set up xdp, xsk pool, rx resize), we call
napi_disable()on the receive queue's napi. In delayedrefill_work, it also callsnapi_disable()on the receive queue's napi. Whennapi_disable()is called on an already disabled napi, it will sleep innapi_disable_lockedwhile still holding thenetdev_lock. As a result, laternapi_enablegets stuck too as it cannot acquire thenetdev_lock. This leads torefill_workand the pause-then-resume tx are stuck altogether.This scenario can be reproducible by binding a XDP socket to virtio-net interface without setting up the fill ring. As a result,
try_fill_recvwill fail until the fill ring is set up andrefill_workis scheduled.This fix adds virtnet_rx_(pause/resume)_all helpers and fixes up the
virtnet_rx_resumeto disable future and cancel all inflights delayedrefill_workbefore callingnapi_disable()to pause the rx.
Quoted 8 lines
Version 3 changes:
- Patch 1: refactor to avoid code duplicationVersion 2 changes:
- Add selftest for deadlock scenarioThanks,
Quang Minh.
Quoted 23 lines
Bui Quang Minh (3):
virtio-net: disable delayed refill when pausing rx
selftests: net: movexdp_helperto net/lib
selftests: net: add avirtio_netdeadlock selftestdrivers/net/virtio_net.c | 69 +++++++++++++++---- tools/testing/selftests/Makefile | 2 +- tools/testing/selftests/drivers/net/Makefile | 2 - tools/testing/selftests/drivers/net/queues.py | 4 +- .../selftests/drivers/net/virtio_net/Makefile | 2 + .../selftests/drivers/net/virtio_net/config | 1 + .../drivers/net/virtio_net/lib/py/__init__.py | 16 +++++ .../drivers/net/virtio_net/xsk_pool.py | 52 ++++++++++++++ tools/testing/selftests/net/lib/.gitignore | 1 + tools/testing/selftests/net/lib/Makefile | 1 + .../{drivers/net => net/lib}/xdp_helper.c | 0 11 files changed, 133 insertions(+), 17 deletions(-) create mode 100644 tools/testing/selftests/drivers/net/virtio_net/lib/py/__init__.py create mode 100755 tools/testing/selftests/drivers/net/virtio_net/xsk_pool.py rename tools/testing/selftests/{drivers/net => net/lib}/xdp_helper.c (100%)Signature · 1 line
2.43.0