hv_netvsc shutdown redo
Subsystems (3)
cover hv_netvsc shutdown redo
These patches change how teardown of Hyper-V network devices is done. These are tested on WS2012 and WS2016.
It moves the tx/rx shutdown into the rndis close handling, and that makes earlier gpadl changes unnecsssary.
Stephen Hemminger (2):
hv_netvsc: make sure device is idle before changes Revert "hv_netvsc: netvsc_teardown_gpadl() split"
drivers/net/hyperv/hyperv_net.h | 1 - drivers/net/hyperv/netvsc.c | 75 ++++++++++++++++++--------------------- drivers/net/hyperv/netvsc_drv.c | 29 ++++++++------- drivers/net/hyperv/rndis_filter.c | 48 ++++++++++++------------- 4 files changed, 73 insertions(+), 80 deletions(-)
Signature · 1 line
2.15.1
RFC 1/2 hv_netvsc: make sure device is idle before changes
Make sure that device is in detached state before doing ring and mtu changes. When doing these changes, wait for all outstanding send completions and ring buffer events.
--- drivers/net/hyperv/hyperv_net.h | 1 - drivers/net/hyperv/netvsc.c | 6 +---- drivers/net/hyperv/netvsc_drv.c | 29 +++++++++++++---------- drivers/net/hyperv/rndis_filter.c | 48 ++++++++++++++++++--------------------- 4 files changed, 40 insertions(+), 44 deletions(-)
drivers/net/hyperv/hyperv_net.h +0 -1
@@ -211,7 +211,6 @@ void netvsc_channel_cb(void *context);int netvsc_poll (struct napi_struct *napi,int budget);void rndis_set_subchannel (struct work_struct *w); -bool rndis_filter_opened (const struct netvsc_device *nvdev);int rndis_filter_open (struct netvsc_device *nvdev);int rndis_filter_close (struct netvsc_device *nvdev); struct netvsc_device *rndis_filter_device_add(struct hv_device *dev,
drivers/net/hyperv/netvsc.c +1 -5
@@ -849,7 +849,7 @@ int netvsc_send(struct net_device *ndev,bool try_batch, xmit_more;/* If device is rescinded, return error and packet will get dropped. */ -if (unlikely (!net_device || net_device->destroy)) +if (unlikely (!net_device))return -ENODEV ;/* We may race with netvsc_connect_vsp()/netvsc_init_buf() and get
@@ -996,10 +996,6 @@ static int send_recv_completions(struct net_device *ndev, mrc->first =0 ; } -/* receive completion ring has been emptied */ -if (unlikely (nvdev->destroy)) -wake_up (&nvdev->wait_drain); -return 0 ; }
drivers/net/hyperv/netvsc_drv.c +17 -12
@@ -820,7 +820,7 @@ static int netvsc_set_channels(struct net_device *net, channels->rx_count || channels->tx_count || channels->other_count)return -EINVAL ; -if (!nvdev || nvdev->destroy) +if (!nvdev)return -ENODEV ;if (nvdev->nvsp_version <NVSP_PROTOCOL_VERSION_5 )
@@ -830,9 +830,6 @@ static int netvsc_set_channels(struct net_device *net,return -EINVAL ; orig = nvdev->num_chn; - was_opened =rndis_filter_opened (nvdev); -if (was_opened) -rndis_filter_close (nvdev);memset (&device_info,0 ,sizeof (device_info)); device_info.num_chn = count;
@@ -841,6 +838,11 @@ static int netvsc_set_channels(struct net_device *net, device_info.recv_sections = nvdev->recv_section_cnt; device_info.recv_section_size = nvdev->recv_section_size; + was_opened =netif_running (net); +netif_device_detach (net); +if (was_opened) +rndis_filter_close (nvdev); +rndis_filter_device_remove (dev, nvdev); nvdev =rndis_filter_device_add (dev, &device_info);
@@ -859,6 +861,8 @@ static int netvsc_set_channels(struct net_device *net,if (was_opened)rndis_filter_open (nvdev); +netif_device_attach (net); +/* We may have missed link change notifications */ net_device_ctx->last_reconfig =0 ;schedule_delayed_work (&net_device_ctx->dwork,0 );
@@ -934,7 +938,7 @@ static int netvsc_change_mtu(struct net_device *ndev, int mtu)bool was_opened;int ret =0 ; -if (!nvdev || nvdev->destroy) +if (!nvdev)return -ENODEV ;/* Change MTU of underlying VF netdev first. */
@@ -944,11 +948,6 @@ static int netvsc_change_mtu(struct net_device *ndev, int mtu)return ret; } -netif_device_detach (ndev); - was_opened =rndis_filter_opened (nvdev); -if (was_opened) -rndis_filter_close (nvdev); -memset (&device_info,0 ,sizeof (device_info)); device_info.num_chn = nvdev->num_chn; device_info.send_sections = nvdev->send_section_cnt;
@@ -956,6 +955,11 @@ static int netvsc_change_mtu(struct net_device *ndev, int mtu) device_info.recv_sections = nvdev->recv_section_cnt; device_info.recv_section_size = nvdev->recv_section_size; + was_opened =netif_running (ndev); +netif_device_detach (ndev); +if (was_opened) +rndis_filter_close (nvdev); +rndis_filter_device_remove (hdev, nvdev); ndev->mtu = mtu;
@@ -1497,7 +1501,7 @@ static int netvsc_set_ringparam(struct net_device *ndev,bool was_opened;int ret =0 ; -if (!nvdev || nvdev->destroy) +if (!nvdev)return -ENODEV ;memset (&orig,0 ,sizeof (orig));
@@ -1519,8 +1523,8 @@ static int netvsc_set_ringparam(struct net_device *ndev, device_info.recv_sections = new_rx; device_info.recv_section_size = nvdev->recv_section_size; + was_opened =netif_running (ndev);netif_device_detach (ndev); - was_opened =rndis_filter_opened (nvdev);if (was_opened)rndis_filter_close (nvdev);
@@ -1542,6 +1546,7 @@ static int netvsc_set_ringparam(struct net_device *ndev,if (was_opened)rndis_filter_open (nvdev); +netif_device_attach (ndev);/* We may have missed link change notifications */
drivers/net/hyperv/rndis_filter.c +22 -26
96 lines not on this page · raw message
Signature · 1 line
2.15.1
RFC 2/2 Revert "hv_netvsc: netvsc_teardown_gpadl() split"
In reply to Stephen HemmingerThis reverts commit 0cf737808ae7cb25e952be619db46b9147a92f46.
The problem that the previous commit was trying to solve was that undoing the mapping of the receive buffer after revoke was problematic. This was because the shutdown logic was not ensuring that there were no receive and sends in flight. The changes in commit 30de1885e897 ("hv_netvsc: make sure device is idle before changes") ensure that device is completely idle.
Windows Server 2012 does not allow the receive buffer mapping to be undone after the channel is closed. This because it assumes when channel is closed, guest won't use it.
--- drivers/net/hyperv/netvsc.c | 69 ++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 36 deletions(-)
drivers/net/hyperv/netvsc.c +33 -36
@@ -101,11 +101,12 @@ static void free_netvsc_device_rcu(struct netvsc_device *nvdev)call_rcu (&nvdev->rcu, free_netvsc_device); } -static void netvsc_revoke_buf (struct hv_device *device, -struct netvsc_device *net_device) +static void netvsc_destroy_buf (struct hv_device *device) {struct nvsp_message *revoke_packet;struct net_device *ndev =hv_get_drvdata (device); +struct net_device_context *ndc =netdev_priv (ndev); +struct netvsc_device *net_device =rtnl_dereference (ndc->nvdev);int ret;/*
@@ -148,6 +149,28 @@ static void netvsc_revoke_buf(struct hv_device *device, net_device->recv_section_cnt =0 ; } +/* Teardown the gpadl on the vsp end */ +if (net_device->recv_buf_gpadl_handle) { + ret =vmbus_teardown_gpadl (device->channel, + net_device->recv_buf_gpadl_handle); + +/* If we failed here, we might as well return and have a leak +* rather than continue and a bugchk +*/ +if (ret !=0 ) { +netdev_err (ndev, +"unable to teardown receive buffer's gpadl\n" ); +return ; + } + net_device->recv_buf_gpadl_handle =0 ; + } + +if (net_device->recv_buf) { +/* Free up the receive buffer */ +vfree (net_device->recv_buf); + net_device->recv_buf =NULL ; + } +/* Deal with the send buffer we may have setup. * If we got a send section size, it means we received a * NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE msg (ie sent
@@ -188,35 +211,7 @@ static void netvsc_revoke_buf(struct hv_device *device, } net_device->send_section_cnt =0 ; } -} - -static void netvsc_teardown_gpadl (struct hv_device *device, -struct netvsc_device *net_device) -{ -struct net_device *ndev =hv_get_drvdata (device); -int ret; - -if (net_device->recv_buf_gpadl_handle) { - ret =vmbus_teardown_gpadl (device->channel, - net_device->recv_buf_gpadl_handle); - -/* If we failed here, we might as well return and have a leak -* rather than continue and a bugchk -*/ -if (ret !=0 ) { -netdev_err (ndev, -"unable to teardown receive buffer's gpadl\n" ); -return ; - } - net_device->recv_buf_gpadl_handle =0 ; - } - -if (net_device->recv_buf) { -/* Free up the receive buffer */ -vfree (net_device->recv_buf); - net_device->recv_buf =NULL ; - } - +/* Teardown the gpadl on the vsp end */ if (net_device->send_buf_gpadl_handle) { ret =vmbus_teardown_gpadl (device->channel, net_device->send_buf_gpadl_handle);
@@ -431,8 +426,7 @@ static int netvsc_init_buf(struct hv_device *device, goto exit; cleanup: -netvsc_revoke_buf (device, net_device); -netvsc_teardown_gpadl (device, net_device); +netvsc_destroy_buf (device); exit:return ret;
@@ -551,6 +545,11 @@ static int netvsc_connect_vsp(struct hv_device *device,return ret; } +static void netvsc_disconnect_vsp (struct hv_device *device) +{ +netvsc_destroy_buf (device); +} +/* * netvsc_device_remove - Callback when the root bus device is removed */
@@ -564,7 +563,7 @@ void netvsc_device_remove(struct hv_device *device)cancel_work_sync (&net_device->subchan_work); -netvsc_revoke_buf (device, net_device); +netvsc_disconnect_vsp (device);RCU_INIT_POINTER (net_device_ctx->nvdev,NULL );
@@ -577,8 +576,6 @@ void netvsc_device_remove(struct hv_device *device)/* Now, we can close the channel safely */ vmbus_close (device->channel); -netvsc_teardown_gpadl (device, net_device); -/* And dissassociate NAPI context from device */ for (i =0 ; i < net_device->num_chn; i++)netif_napi_del (&net_device->chan_table[i].napi);
Signature · 1 line
2.15.1
Quoted 6 lines · Stephen Hemminger
These patches change how teardown of Hyper-V network devices is done. These are tested on WS2012 and WS2016.
It moves the tx/rx shutdown into the rndis close handling, and that makes earlier gpadl changes unnecsssary.
Thank you Stephen,
I gave these a try and they didn't survive my 'death row' test on
WS2016: I run 3 things in parallel:
1) iperf to some external IP
2) while true; do ethtool -L ethX combined 6; ethtool -L ethX combined 8; done
3) while true; do ip link set dev ethX mtu 1400; ip link set dev ethX mtu 1450; done
I ended up with a hang:
[ 1226.710034] INFO: task ip:2357 blocked for more than 120 seconds. [ 1226.712397] Not tainted 4.15.0-rc9+ #321 [ 1226.714030] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1226.716724] ip D 0 2357 1474 0x00000000 [ 1226.718698] Call Trace: [ 1226.719588] ? __schedule+0x1da/0x7b0 [ 1226.720910] ? get_page_from_freelist+0x106d/0x15c0 [ 1226.722648] schedule+0x28/0x80 [ 1226.723807] schedule_preempt_disabled+0xa/0x10 [ 1226.725952] __mutex_lock.isra.1+0x1a0/0x4e0 [ 1226.727915] ? rtnetlink_rcv_msg+0x212/0x2d0 [ 1226.729849] rtnetlink_rcv_msg+0x212/0x2d0 [ 1226.731611] ? rtnl_calcit.isra.28+0x110/0x110 [ 1226.733824] netlink_rcv_skb+0x4a/0x120 [ 1226.736916] netlink_unicast+0x19d/0x250 [ 1226.738907] netlink_sendmsg+0x2a5/0x3a0 [ 1226.740762] sock_sendmsg+0x30/0x40 [ 1226.742552] SYSC_sendto+0x10e/0x140 [ 1226.744310] ? __do_page_fault+0x26d/0x4c0 [ 1226.746332] entry_SYSCALL_64_fastpath+0x20/0x83 [ 1226.748730] RIP: 0033:0x7ff2cdc9aa7d [ 1226.750776] RSP: 002b:00007ffd0a3455e8 EFLAGS: 00000246 [ 1349.590041] INFO: task kworker/3:6:1586 blocked for more than 120 seconds. [ 1349.595358] Not tainted 4.15.0-rc9+ #321 [ 1349.597335] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1349.600638] kworker/3:6 D 0 1586 2 0x80000000 [ 1349.603335] Workqueue: ipv6_addrconf addrconf_verify_work [ 1349.605779] Call Trace: [ 1349.607080] ? __schedule+0x1da/0x7b0 [ 1349.608856] ? update_load_avg+0x563/0x6d0 [ 1349.610834] ? update_curr+0xb9/0x190 [ 1349.613050] schedule+0x28/0x80 [ 1349.615290] schedule_preempt_disabled+0xa/0x10 [ 1349.617306] __mutex_lock.isra.1+0x1a0/0x4e0 [ 1349.619072] ? addrconf_verify_work+0xa/0x20 [ 1349.621108] addrconf_verify_work+0xa/0x20 [ 1349.623107] process_one_work+0x188/0x380 [ 1349.625012] worker_thread+0x2e/0x390 [ 1349.626976] ? process_one_work+0x380/0x380 [ 1349.628925] kthread+0x111/0x130 [ 1349.630498] ? kthread_create_worker_on_cpu+0x70/0x70 [ 1349.632786] ? do_group_exit+0x3a/0xa0 [ 1349.634598] ret_from_fork+0x35/0x40
....
(I'm not 100% sure this is a _new_ issue btw, it can happen that the race was always there and it's just easier to trigger it now).
I'll try to do more testing next week.
Thanks,
Signature · 1 line
Vitaly
A message that is not in the archive (sent off-list or not yet received): SN4PR2101MB087887EE5C948A92068A4F17CAE70@SN4PR2101MB0878.namprd21.prod.outlook.com
Quoted 2 lines · On Sat, 27 Jan 2018 21:00:12 +0000 Haiyang Zhang <[email protected]> wrote:
In the functions,
set_channelsandchange_mtu, we used to callnetvsc_closewhich has a wait for ring buffers to drain. Now, we callrndis_filter_close()directly without the wait for rings to drain. Could this be a problem?
rndis_filter_close now waits for rings to drain.