net/vdev_netvsc: handle removal of associated pci device
PATCH 1/1 net/vdev_netvsc: handle removal of associated pci device
The vdev_netvsc was not detecting when the associated PCI device (SRIOV) was removed. Because of that it would keep feeding the same (removed) device to failsafe PMD which would then unsuccessfully try and probe for it.
Change to use a mark/sweep method to detect that PCI device was removed, and also only tell failsafe about new PCI devices. Vdev_netvsc does not have to keep stuffing the pipe with the same already existing PCI device.
--- drivers/net/vdev_netvsc/vdev_netvsc.c | 39 +++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-)
drivers/net/vdev_netvsc/vdev_netvsc.c +34 -5
@@ -66,6 +66,7 @@ struct vdev_netvsc_ctx {char devargs[256 ];/**< Fail-safe device arguments. */ char if_name[IF_NAMESIZE ];/**< NetVSC netdevice name. */ unsigned int if_index;/**< NetVSC netdevice index. */ +int pci_found;/**< Device detection */ struct rte_ether_addr if_addr;/**< NetVSC MAC address. */ int pipe[2 ];/**< Fail-safe communication pipe. */ char yield[256 ];/**< PCI sub-device arguments. */
@@ -405,11 +406,18 @@ vdev_netvsc_device_probe(const struct if_nameindex *iface, len =strlen (addr);if (!len)return 0 ; + + ctx->pci_found =1 ; + +/* Skip if this is same device already sent to failsafe */ +if (strcmp (addr, ctx->yield) ==0 ) +return 0 ; + +DRV_LOG (DEBUG ,"associating PCI device \"%s\" with NetVSC" +" interface \"%s\" (index %u)" , addr, ctx->if_name, + ctx->if_index); +/* Send PCI device argument to fail-safe PMD instance. */ -if (strcmp (addr, ctx->yield)) -DRV_LOG (DEBUG ,"associating PCI device \"%s\" with NetVSC" -" interface \"%s\" (index %u)" , addr, ctx->if_name, - ctx->if_index);memmove (buf, addr, len +1 ); addr = buf; buf[len] ='\n' ;
@@ -452,12 +460,33 @@ vdev_netvsc_alarm(__rte_unused void *arg)struct vdev_netvsc_ctx *ctx;int ret; +/* 1st pass: clear PCI flag on all devices */ + LIST_FOREACH(ctx, &vdev_netvsc_ctx_list, entry) { + ctx->pci_found =0 ; + } + +/* 2nd pass: scan all system devices to look for changes to this device */ LIST_FOREACH(ctx, &vdev_netvsc_ctx_list, entry) { ret =vdev_netvsc_foreach_iface (vdev_netvsc_device_probe,0 , ctx); -if (ret <0 ) + +if (ret <0 ) { +DRV_LOG (NOTICE ,"can not scan devices for %s\n" , + ctx->if_name);break ; + } } + +/* 3rd pass: detect PCI removal */ + LIST_FOREACH(ctx, &vdev_netvsc_ctx_list, entry) { +if (!ctx->pci_found && ctx->yield[0 ]) { +DRV_LOG (DEBUG ,"disassociating PCI device \"%s\" from NetVSC" +" interface \"%s\" (index %u)" , ctx->yield, ctx->if_name, + ctx->if_index); + ctx->yield[0 ] ='\0' ; + } + } +if (!vdev_netvsc_ctx_count)return ; ret = rte_eal_alarm_set(VDEV_NETVSC_PROBE_MS * 1000,
Signature · 1 line
2.27.0
Quoted 8 lines
Subject: [dpdk-dev] [PATCH]
net/vdev_netvsc: handle removal of associated pci deviceThe
vdev_netvscwas not detecting when the associated PCI device (SRIOV) was removed. Because of that it would keep feeding the same (removed) device to failsafe PMD which would then unsuccessfully try and probe for it.
Quoted 8 lines
Change to use a mark/sweep method to detect that PCI device was removed, and also only tell failsafe about new PCI devices.
Vdev_netvscdoes not have to keep stuffing the pipe with the same already existing PCI device.Fixes e7dc5d7becc5 ("net/vdev_netvsc: implement core functionality")Signed-off-by Stephen Hemminger <[email protected]>
Quoted 85 lines
---
drivers/net/vdev_netvsc/vdev_netvsc.c| 39 +++++++++++++++++++++++-- --1 file changed, 34 insertions(+), 5 deletions(-)drivers/net/vdev_netvsc/vdev_netvsc.c +1
-0@@ -66,6 +66,7 @@ struct vdev_netvsc_ctx {char devargs[256 ];/**< Fail-safe device arguments. */ char if_name[IF_NAMESIZE ];/**< NetVSC netdevice name. */ unsigned int if_index;/**< NetVSC netdevice index. */ +int pci_found;/**< Device detection */ struct rte_ether_addr if_addr;/**< NetVSC MAC address. */ int pipe[2 ];/**< Fail-safe communication pipe. */ char yield[256]; /**< PCI sub-device arguments. */drivers/net/vdev_netvsc/vdev_netvsc.c +11
-2@@ -405,11 +406,18 @@ vdev_netvsc_device_probe(const structif_nameindex *iface, len =strlen (addr);if (!len)return 0 ; + + ctx->pci_found =1 ; + +/* Skip if this is same device already sent to failsafe */ +if (strcmp (addr, ctx->yield) ==0 ) +return 0 ; + +DRV_LOG (DEBUG ,"associating PCI device \"%s\" with NetVSC" +" interface \"%s\" (index %u)" , addr, ctx->if_name, + ctx->if_index); +/* Send PCI device argument to fail-safe PMD instance. */ -if (strcmp (addr, ctx->yield)) - DRV_LOG(DEBUG, "associating PCI device \"%s\" withNetVSC"
drivers/net/vdev_netvsc/vdev_netvsc.c +0
-2-" interface \"%s\" (index %u)" , addr, ctx->if_name, - ctx->if_index);memmove(buf, addr, len + 1); addr = buf; buf[len] = '\n';drivers/net/vdev_netvsc/vdev_netvsc.c +17
-1@@ -452,12 +460,33 @@ vdev_netvsc_alarm(__rte_unused void *arg)struct vdev_netvsc_ctx *ctx;int ret; +/* 1st pass: clear PCI flag on all devices */ + LIST_FOREACH(ctx, &vdev_netvsc_ctx_list, entry) { + ctx->pci_found =0 ; + } + +/* 2nd pass: scan all system devices to look for changes to this +device */ LIST_FOREACH(ctx, &vdev_netvsc_ctx_list, entry) { ret =vdev_netvsc_foreach_iface (vdev_netvsc_device_probe,0 , ctx); -if (ret <0 ) + +if (ret <0 ) { +DRV_LOG (NOTICE ,"can not scan devices for %s\n" , + ctx->if_name);break ; + } } + +/* 3rd pass: detect PCI removal */ + LIST_FOREACH(ctx, &vdev_netvsc_ctx_list, entry) { +if (!ctx->pci_found && ctx->yield[0 ]) { + DRV_LOG(DEBUG, "disassociating PCI device \"%s\"from NetVSC"
drivers/net/vdev_netvsc/vdev_netvsc.c +1
-0+ " interface \"%s\" (index %u)", ctx->yield, ctx-Quoted 1 line
if_name,drivers/net/vdev_netvsc/vdev_netvsc.c +5
-0+ ctx->if_index); + ctx->yield[0 ] ='\0' ; + } + } +if (!vdev_netvsc_ctx_count) return; ret = rte_eal_alarm_set(VDEV_NETVSC_PROBE_MS * 1000,--
2.27.0
Hi Stephen
From: Stephen Hemminger:
Quoted 9 lines
The
vdev_netvscwas not detecting when the associated PCI device (SRIOV) was removed. Because of that it would keep feeding the same (removed) device to failsafe PMD which would then unsuccessfully try and probe for it.Change to use a mark/sweep method to detect that PCI device was removed, and also only tell failsafe about new PCI devices.
Vdev_netvscdoes not have to keep stuffing the pipe with the same already existing PCI device.
As I know, the vdev_netvsc driver doesn't call to failsafe if the PCI device is not detected by the readlink command(considered as removed)... Am I missing something?
Quoted 23 lines
Fixes e7dc5d7becc5 ("net/vdev_netvsc: implement core functionality")Signed-off-by Stephen Hemminger <[email protected]>--- drivers/net/vdev_netvsc/vdev_netvsc.c | 39 +++++++++++++++++++++++- --- 1 file changed, 34 insertions(+), 5 deletions(-)drivers/net/vdev_netvsc/vdev_netvsc.c +1
-0@@ -66,6 +66,7 @@ struct vdev_netvsc_ctx {char devargs[256 ];/**< Fail-safe device arguments. */ char if_name[IF_NAMESIZE ];/**< NetVSC netdevice name. */ unsigned int if_index;/**< NetVSC netdevice index. */ +int pci_found;/**< Device detection */ struct rte_ether_addr if_addr;/**< NetVSC MAC address. */ int pipe[2 ];/**< Fail-safe communication pipe. */ char yield[256 ];/**< PCI sub-device arguments. */
drivers/net/vdev_netvsc/vdev_netvsc.c vdev_netvsc_device_probe(const struct · line 410
Quoted 5 lines
drivers/net/vdev_netvsc/vdev_netvsc.c +1
-0@@ -405,11 +406,18 @@ vdev_netvsc_device_probe(const structif_nameindex *iface, len =strlen (addr);if (!len)return 0 ; +
The file code convention is to use blank lines in blocks only after variables declarations.
Quoted 57 lines
drivers/net/vdev_netvsc/vdev_netvsc.c +10
-4+ ctx->pci_found =1 ; + +/* Skip if this is same device already sent to failsafe */ +if (strcmp (addr, ctx->yield) ==0 ) +return 0 ; + +DRV_LOG (DEBUG ,"associating PCI device \"%s\" with NetVSC" +" interface \"%s\" (index %u)" , addr, ctx->if_name, + ctx->if_index); +/* Send PCI device argument to fail-safe PMD instance. */ -if (strcmp (addr, ctx->yield)) -DRV_LOG (DEBUG ,"associating PCI device \"%s\" with NetVSC" -" interface \"%s\" (index %u)" , addr, ctx->if_name, - ctx->if_index);memmove (buf, addr, len +1 ); addr = buf;buf[len] = '\n';drivers/net/vdev_netvsc/vdev_netvsc.c +17
-1@@ -452,12 +460,33 @@ vdev_netvsc_alarm(__rte_unused void *arg)struct vdev_netvsc_ctx *ctx;int ret; +/* 1st pass: clear PCI flag on all devices */ + LIST_FOREACH(ctx, &vdev_netvsc_ctx_list, entry) { + ctx->pci_found =0 ; + } + +/* 2nd pass: scan all system devices to look for changes to this +device */ LIST_FOREACH(ctx, &vdev_netvsc_ctx_list, entry) { ret =vdev_netvsc_foreach_iface (vdev_netvsc_device_probe,0 , ctx); -if (ret <0 ) + +if (ret <0 ) { +DRV_LOG (NOTICE ,"can not scan devices for %s\n" , + ctx->if_name);break ; + } } + +/* 3rd pass: detect PCI removal */ + LIST_FOREACH(ctx, &vdev_netvsc_ctx_list, entry) { +if (!ctx->pci_found && ctx->yield[0 ]) { + DRV_LOG(DEBUG, "disassociating PCI device \"%s\" fromNetVSC"
drivers/net/vdev_netvsc/vdev_netvsc.c +6
-1+" interface \"%s\" (index %u)" , ctx->yield, ctx->if_name, + ctx->if_index); + ctx->yield[0 ] ='\0' ; + } + } +if (!vdev_netvsc_ctx_count)return ; ret = rte_eal_alarm_set(VDEV_NETVSC_PROBE_MS * 1000, --2.27.0
Quoted 8 lines · On Sun, 6 Sep 2020 12:38:18 +0000 Matan Azrad <[email protected]> wrote:
Hi Stephen
From: Stephen Hemminger:
Quoted 5 lines
The
vdev_netvscwas not detecting when the associated PCI device (SRIOV) was removed. Because of that it would keep feeding the same (removed) device to failsafe PMD which would then unsuccessfully try and probe for it.
Quoted 7 lines
Quoted 4 lines
Change to use a mark/sweep method to detect that PCI device was removed, and also only tell failsafe about new PCI devices.
Vdev_netvscdoes not have to keep stuffing the pipe with the same already existing PCI device.As I know, the
vdev_netvscdriver doesn't call to failsafe if the PCI device is not detected by the readlink command(considered as removed)... Am I missing something?
The original code is broken because ctx_yield is not cleared, it keeps sending the same value. It looks like device removal and add was never tested.
If you test removal you will see that vdev_netvsc:
1. Sends same PCI device repeatedly to failsafe (every alarm call) This is harmless, but useless.
2. When device is removed, keeps doing #1
Hi Stephen
From: Stephen Hemminger:
Quoted 16 lines
Quoted 13 lines · On Sun, 6 Sep 2020 12:38:18 +0000 Matan Azrad <[email protected]> wrote:
Hi Stephen
From: Stephen Hemminger:
Quoted 9 lines
The
vdev_netvscwas not detecting when the associated PCI device (SRIOV) was removed. Because of that it would keep feeding the same (removed) device to failsafe PMD which would then unsuccessfully try and probe for it.Change to use a mark/sweep method to detect that PCI device was removed, and also only tell failsafe about new PCI devices.
Vdev_netvscdoes not have to keep stuffing the pipe with the same already existing PCI device.
Quoted 6 lines
Quoted 1 line
As I know, the
vdev_netvscdriver doesn't call to failsafe if the PCI device isnot detected by the readlink command(considered as removed)...
Quoted 1 line
Am I missing something?
The original code is broken because
ctx_yieldis not cleared, it keeps sending the same value.
Looking on the code again, It looks like ctx->yield has no effect on the next pipe write, It is just used for log.
After the PCI interface matching to the netvsc interface, the pipe write is triggered only if the readlink commands success to see the plugged-in PCI device:
readlink /sys/class/net/[iface]/device/subsystem shows "pci"
readlink /sys/class/net/[iface]/device shows the pci device ID.
So, the assumption is when the above readlink failed on the interface the device is removed(plugged-out) and the fd write will not happen.
The code will continue to retry probe again and again until success only for plugged-in pci device matched the netvsc device.
Quoted 1 line
It looks like device removal and add was never tested.
This is basic test we have to test plug-in plug-out and it passed every day in the last years.
Maybe something new and special in your setup?
Quoted 4 lines
If you test removal you will see that
vdev_netvsc:
1. Sends same PCI device repeatedly to failsafe (every alarm call) This is harmless, but useless.
2. When device is removed, keeps doing #1
Quoted 40 lines
Subject: Re: [dpdk-dev] [PATCH]
net/vdev_netvsc: handle removal of associated pci deviceHi Stephen
From: Stephen Hemminger:
Quoted 17 lines
Quoted 14 lines · On Sun, 6 Sep 2020 12:38:18 +0000 Matan Azrad <[email protected]> wrote:
Hi Stephen
From: Stephen Hemminger:
Quoted 10 lines
The
vdev_netvscwas not detecting when the associated PCI device (SRIOV) was removed. Because of that it would keep feeding the same
(removed) device to failsafe PMD which would then unsuccessfully try and probe for it.Change to use a mark/sweep method to detect that PCI device was removed, and also only tell failsafe about new PCI devices.
Vdev_netvscdoes not have to keep stuffing the pipe with the same already existing PCI device.Quoted 7 lines
Quoted 2 lines
As I know, the
vdev_netvscdriver doesn't call to failsafe if the PCI device isnot detected by the readlink command(considered as removed)...
Quoted 1 line
Am I missing something?
The original code is broken because
ctx_yieldis not cleared, it keeps sending the same value.Looking on the code again, It looks like
ctx->yieldhas no effect on the next pipe write, It is just used for log.After the PCI interface matching to the netvsc interface, the pipe write is triggered only if the readlink commands success to see the plugged-in PCI device:
readlink /sys/class/net/[iface]/device/subsystem shows "pci" readlink /sys/class/net/[iface]/device shows the pci device ID.
Quoted 5 lines
So, the assumption is when the above readlink failed on the interface the device is removed(plugged-out) and the fd write will not happen.
The code will continue to retry probe again and again until success only for plugged-in pci device matched the netvsc device.
Hi Matan,
The original code keeps writing to pipe even it's the same PCI device. The new code writes to pipe for a new device, only once. See the following code:
+ /* Skip if this is same device already sent to failsafe */
+ if (strcmp(addr, ctx->yield) == 0)
+ return 0;
This patch also saves lots of CPU since it no longer writes to pipe all the time. You are correct about the code will continue to probe on a new PCI device. But someone has to do it to handle hot-add.
Thanks,
Long
Quoted 12 lines
Quoted 2 lines
It looks like device removal and add was never tested.
This is basic test we have to test plug-in plug-out and it passed every day in the last years.
Maybe something new and special in your setup?
Quoted 4 lines
If you test removal you will see that
vdev_netvsc:
1. Sends same PCI device repeatedly to failsafe (every alarm call) This is harmless, but useless.
2. When device is removed, keeps doing #1
Hi Li
From: Long Li <[email protected]>
Quoted 44 lines
Quoted 44 lines
Subject: Re: [dpdk-dev] [PATCH]
net/vdev_netvsc: handle removal of associated pci deviceHi Stephen
From: Stephen Hemminger:
Quoted 17 lines
Quoted 14 lines · On Sun, 6 Sep 2020 12:38:18 +0000 Matan Azrad <[email protected]> wrote:
Hi Stephen
From: Stephen Hemminger:
Quoted 10 lines
The
vdev_netvscwas not detecting when the associated PCI device (SRIOV) was removed. Because of that it would keep feeding the same
(removed) device to failsafe PMD which would then unsuccessfully try and probe for it.Change to use a mark/sweep method to detect that PCI device was removed, and also only tell failsafe about new PCI devices.
Vdev_netvscdoes not have to keep stuffing the pipe with the same already existing PCI device.Quoted 7 lines
Quoted 2 lines
As I know, the
vdev_netvscdriver doesn't call to failsafe if the PCI device isnot detected by the readlink command(considered as removed)...
Quoted 1 line
Am I missing something?
The original code is broken because
ctx_yieldis not cleared, it keeps sending the same value.Looking on the code again, It looks like
ctx->yieldhas no effect on the next pipe write, It is just used for log.After the PCI interface matching to the netvsc interface, the pipe write is triggered only if the readlink commands success to see the plugged-in PCI
device:
readlink /sys/class/net/[iface]/device/subsystem shows "pci" readlink /sys/class/net/[iface]/device shows the pci device ID.So, the assumption is when the above readlink failed on the interface the device is removed(plugged-out) and the fd write will not happen.
Quoted 6 lines
Quoted 2 lines
The code will continue to retry probe again and again until success only for plugged-in pci device matched the netvsc device.
Hi Matan,
The original code keeps writing to pipe even it's the same PCI device.
Yes, the vdev_netvsc writes any plugged-in device to the associated netvsc device fd.
drivers/net/vdev_netvsc/vdev_netvsc.c vdev_netvsc_device_probe(const struct if_nameindex *iface, · line 414
Quoted 7 lines
The
new code writes to pipe for a new device, only once. See the following code:drivers/net/vdev_netvsc/vdev_netvsc.c +3
-0+/* Skip if this is same device already sent to failsafe */ +if (strcmp (addr, ctx->yield) ==0 ) +return 0 ;
I understand you want to optimize the pipe writing to be written only after plugged-in hot event.
The current solution suffers from race: the PCI device may be plugged-out and plugged-in in short time shorter than the driver alarm delay, then the PCI device plugged-in detection will lost.
My suggestion:
Add validation to the plugged-in device probing state and that it is owned by failsafe(using ownership API) - don't write the pipe if so.
Matan
Quoted 20 lines
This patch also saves lots of CPU since it no longer writes to pipe all the time. You are correct about the code will continue to probe on a new PCI device. But someone has to do it to handle hot-add.
Thanks,
LongQuoted 12 lines
Quoted 2 lines
It looks like device removal and add was never tested.
This is basic test we have to test plug-in plug-out and it passed every day in the last years.
Maybe something new and special in your setup?
Quoted 4 lines
If you test removal you will see that
vdev_netvsc:
1. Sends same PCI device repeatedly to failsafe (every alarm call) This is harmless, but useless.
2. When device is removed, keeps doing #1
HI Matan,
While troubleshooting a failure in DPDK on device removal when VF device briefly disappears and comes back, I notice the failsafe driver is trying repeatedly to start a sub device (after this sub device has been successfully configured, but later hot removed from the kernel). This is due to repeated alarms calling fs_dev_start(). I trace into this commit:
ae80146 net/failsafe: fix removed device handling
The implementation of fs_err() is interesting:
+fs_err(struct sub_device *sdev, int err)
+{
+ /* A device removal shouldn't be reported as an error. */
+ if (sdev->remove == 1 || err == -EIO)
+ return rte_errno = 0;
+ return err;
+}
If I change this function to:
@@ -497,7 +497,7 @@ int failsafe_eth_new_event_callback(uint16_t port_id
fs_err(struct sub_device *sdev, int err)
{
/* A device removal shouldn't be reported as an error. */
- if (sdev->remove == 1 || err == -EIO)
+ if (sdev->remove == 1 && err == -EIO)
return rte_errno = 0;
return err;}
The hung is going away. I don't know the reason why we use a || in the if(). If a call to rte_eth_dev_start() returning EIO (as the case in fs_dev_start), the best choice would be bail out and fail this sub device.
Can you please take a look?
Thanks,
Long
________________________________
From: Matan Azrad <[email protected]>
Sent: Tuesday, September 15, 2020 12:00 AM
To: Long Li <[email protected]>; Stephen Hemminger <[email protected]>
Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Raslan Darawsheh <[email protected]>
Subject: RE: [dpdk-dev] [PATCH] net/vdev_netvsc: handle removal of associated pci device
Hi Li
From: Long Li <[email protected]>
Quoted 44 lines
Quoted 44 lines
Subject: Re: [dpdk-dev] [PATCH]
net/vdev_netvsc: handle removal of associated pci deviceHi Stephen
From: Stephen Hemminger:
Quoted 17 lines
Quoted 14 lines · On Sun, 6 Sep 2020 12:38:18 +0000 Matan Azrad <[email protected]> wrote:
Hi Stephen
From: Stephen Hemminger:
Quoted 10 lines
The
vdev_netvscwas not detecting when the associated PCI device (SRIOV) was removed. Because of that it would keep feeding the same
(removed) device to failsafe PMD which would then unsuccessfully try and probe for it.Change to use a mark/sweep method to detect that PCI device was removed, and also only tell failsafe about new PCI devices.
Vdev_netvscdoes not have to keep stuffing the pipe with the same already existing PCI device.Quoted 7 lines
Quoted 2 lines
As I know, the
vdev_netvscdriver doesn't call to failsafe if the PCI device isnot detected by the readlink command(considered as removed)...
Quoted 1 line
Am I missing something?
The original code is broken because
ctx_yieldis not cleared, it keeps sending the same value.Looking on the code again, It looks like
ctx->yieldhas no effect on the next pipe write, It is just used for log.After the PCI interface matching to the netvsc interface, the pipe write is triggered only if the readlink commands success to see the plugged-in PCI
device:
readlink /sys/class/net/[iface]/device/subsystem shows "pci" readlink /sys/class/net/[iface]/device shows the pci device ID.So, the assumption is when the above readlink failed on the interface the device is removed(plugged-out) and the fd write will not happen.
Quoted 6 lines
Quoted 2 lines
The code will continue to retry probe again and again until success only for plugged-in pci device matched the netvsc device.
Hi Matan,
The original code keeps writing to pipe even it's the same PCI device.
Yes, the vdev_netvsc writes any plugged-in device to the associated netvsc device fd.
drivers/net/vdev_netvsc/vdev_netvsc.c vdev_netvsc_device_probe(const struct if_nameindex *iface, · line 414
Quoted 7 lines
The
new code writes to pipe for a new device, only once. See the following code:drivers/net/vdev_netvsc/vdev_netvsc.c +3
-0+/* Skip if this is same device already sent to failsafe */ +if (strcmp (addr, ctx->yield) ==0 ) +return 0 ;
I understand you want to optimize the pipe writing to be written only after plugged-in hot event.
The current solution suffers from race: the PCI device may be plugged-out and plugged-in in short time shorter than the driver alarm delay, then the PCI device plugged-in detection will lost.
My suggestion:
Add validation to the plugged-in device probing state and that it is owned by failsafe(using ownership API) - don't write the pipe if so.
Matan
Quoted 20 lines
This patch also saves lots of CPU since it no longer writes to pipe all the time. You are correct about the code will continue to probe on a new PCI device. But someone has to do it to handle hot-add.
Thanks,
LongQuoted 12 lines
Quoted 2 lines
It looks like device removal and add was never tested.
This is basic test we have to test plug-in plug-out and it passed every day in the last years.
Maybe something new and special in your setup?
Quoted 4 lines
If you test removal you will see that
vdev_netvsc:
1. Sends same PCI device repeatedly to failsafe (every alarm call) This is harmless, but useless.
2. When device is removed, keeps doing #1
Gaetan, Matan,
Please could you check?
25/09/2020 22:30, Long Li:
Quoted 137 lines
HI Matan,
While troubleshooting a failure in DPDK on device removal when VF device briefly disappears and comes back, I notice the failsafe driver is trying repeatedly to start a sub device (after this sub device has been successfully configured, but later hot removed from the kernel). This is due to repeated alarms calling
fs_dev_start(). I trace into this commit:ae80146 net/failsafe: fix removed device handling
The implementation of
fs_err()is interesting:(quoted diff) +7
-0+fs_err(struct sub_device *sdev, int err) +{ + /* A device removal shouldn't be reported as an error. */ + if (sdev->remove == 1 || err == -EIO) + return rte_errno = 0; + return err; +}If I change this function to:
(quoted diff) +1
-1@@ -497,7 +497,7 @@ int failsafe_eth_new_event_callback(uint16_t port_id fs_err(struct sub_device *sdev, int err) { /* A device removal shouldn't be reported as an error. */ - if (sdev->remove == 1 || err == -EIO) + if (sdev->remove == 1 && err == -EIO) return rte_errno = 0; return err; }The hung is going away. I don't know the reason why we use a || in the
if(). If a call torte_eth_dev_start()returning EIO (as the case infs_dev_start), the best choice would be bail out and fail this sub device.Can you please take a look?
Thanks,
Long________________________________
From: Matan Azrad <[email protected]>
Sent: Tuesday, September 15, 2020 12:00 AM
To: Long Li <[email protected]>; Stephen Hemminger <[email protected]>
Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Raslan Darawsheh <[email protected]>
Subject: RE: [dpdk-dev] [PATCH]net/vdev_netvsc: handle removal of associated pci deviceHi Li
From: Long Li <[email protected]>
Quoted 44 lines
Quoted 44 lines
Subject: Re: [dpdk-dev] [PATCH]
net/vdev_netvsc: handle removal of associated pci deviceHi Stephen
From: Stephen Hemminger:
Quoted 17 lines
Quoted 14 lines · On Sun, 6 Sep 2020 12:38:18 +0000 Matan Azrad <[email protected]> wrote:
Hi Stephen
From: Stephen Hemminger:
Quoted 10 lines
The
vdev_netvscwas not detecting when the associated PCI device (SRIOV) was removed. Because of that it would keep feeding the same
(removed) device to failsafe PMD which would then unsuccessfully try and probe for it.Change to use a mark/sweep method to detect that PCI device was removed, and also only tell failsafe about new PCI devices.
Vdev_netvscdoes not have to keep stuffing the pipe with the same already existing PCI device.Quoted 7 lines
Quoted 2 lines
As I know, the
vdev_netvscdriver doesn't call to failsafe if the PCI device isnot detected by the readlink command(considered as removed)...
Quoted 1 line
Am I missing something?
The original code is broken because
ctx_yieldis not cleared, it keeps sending the same value.Looking on the code again, It looks like
ctx->yieldhas no effect on the next pipe write, It is just used for log.After the PCI interface matching to the netvsc interface, the pipe write is triggered only if the readlink commands success to see the plugged-in PCI
device:
readlink /sys/class/net/[iface]/device/subsystem shows "pci" readlink /sys/class/net/[iface]/device shows the pci device ID.So, the assumption is when the above readlink failed on the interface the device is removed(plugged-out) and the fd write will not happen.
Quoted 6 lines
Quoted 2 lines
The code will continue to retry probe again and again until success only for plugged-in pci device matched the netvsc device.
Hi Matan,
The original code keeps writing to pipe even it's the same PCI device.
Yes, the
vdev_netvscwrites any plugged-in device to the associated netvsc device fd.Review
Quoted 7 lines
The
new code writes to pipe for a new device, only once. See the following code:(quoted diff) +3
-0+ /* Skip if this is same device already sent to failsafe */ + if (strcmp(addr, ctx->yield) == 0) + return 0;I understand you want to optimize the pipe writing to be written only after plugged-in hot event.
The current solution suffers from race: the PCI device may be plugged-out and plugged-in in short time shorter than the driver alarm delay, then the PCI device plugged-in detection will lost.
My suggestion:
Add validation to the plugged-in device probing state and that it is owned by failsafe(using ownership API) - don't write the pipe if so.Matan
Quoted 20 lines
This patch also saves lots of CPU since it no longer writes to pipe all the time. You are correct about the code will continue to probe on a new PCI device. But someone has to do it to handle hot-add.
Thanks,
LongQuoted 12 lines
Quoted 2 lines
It looks like device removal and add was never tested.
This is basic test we have to test plug-in plug-out and it passed every day in the last years.
Maybe something new and special in your setup?
Quoted 4 lines
If you test removal you will see that
vdev_netvsc:
1. Sends same PCI device repeatedly to failsafe (every alarm call) This is harmless, but useless.
2. When device is removed, keeps doing #1
Fixing Gaetan's address
20/10/2020 00:33, Thomas Monjalon:
Quoted 145 lines
Gaetan, Matan,
Please could you check?25/09/2020 22:30, Long Li:
Quoted 137 lines
HI Matan,
While troubleshooting a failure in DPDK on device removal when VF device briefly disappears and comes back, I notice the failsafe driver is trying repeatedly to start a sub device (after this sub device has been successfully configured, but later hot removed from the kernel). This is due to repeated alarms calling
fs_dev_start(). I trace into this commit:ae80146 net/failsafe: fix removed device handling
The implementation of
fs_err()is interesting:(quoted diff) +7
-0+fs_err(struct sub_device *sdev, int err) +{ + /* A device removal shouldn't be reported as an error. */ + if (sdev->remove == 1 || err == -EIO) + return rte_errno = 0; + return err; +}If I change this function to:
(quoted diff) +1
-1@@ -497,7 +497,7 @@ int failsafe_eth_new_event_callback(uint16_t port_id fs_err(struct sub_device *sdev, int err) { /* A device removal shouldn't be reported as an error. */ - if (sdev->remove == 1 || err == -EIO) + if (sdev->remove == 1 && err == -EIO) return rte_errno = 0; return err; }The hung is going away. I don't know the reason why we use a || in the
if(). If a call torte_eth_dev_start()returning EIO (as the case infs_dev_start), the best choice would be bail out and fail this sub device.Can you please take a look?
Thanks,
Long________________________________
From: Matan Azrad <[email protected]>
Sent: Tuesday, September 15, 2020 12:00 AM
To: Long Li <[email protected]>; Stephen Hemminger <[email protected]>
Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Raslan Darawsheh <[email protected]>
Subject: RE: [dpdk-dev] [PATCH]net/vdev_netvsc: handle removal of associated pci deviceHi Li
From: Long Li <[email protected]>
Quoted 44 lines
Quoted 44 lines
Subject: Re: [dpdk-dev] [PATCH]
net/vdev_netvsc: handle removal of associated pci deviceHi Stephen
From: Stephen Hemminger:
Quoted 17 lines
Quoted 14 lines · On Sun, 6 Sep 2020 12:38:18 +0000 Matan Azrad <[email protected]> wrote:
Hi Stephen
From: Stephen Hemminger:
Quoted 10 lines
The
vdev_netvscwas not detecting when the associated PCI device (SRIOV) was removed. Because of that it would keep feeding the same
(removed) device to failsafe PMD which would then unsuccessfully try and probe for it.Change to use a mark/sweep method to detect that PCI device was removed, and also only tell failsafe about new PCI devices.
Vdev_netvscdoes not have to keep stuffing the pipe with the same already existing PCI device.Quoted 7 lines
Quoted 2 lines
As I know, the
vdev_netvscdriver doesn't call to failsafe if the PCI device isnot detected by the readlink command(considered as removed)...
Quoted 1 line
Am I missing something?
The original code is broken because
ctx_yieldis not cleared, it keeps sending the same value.Looking on the code again, It looks like
ctx->yieldhas no effect on the next pipe write, It is just used for log.After the PCI interface matching to the netvsc interface, the pipe write is triggered only if the readlink commands success to see the plugged-in PCI
device:
readlink /sys/class/net/[iface]/device/subsystem shows "pci" readlink /sys/class/net/[iface]/device shows the pci device ID.So, the assumption is when the above readlink failed on the interface the device is removed(plugged-out) and the fd write will not happen.
Quoted 6 lines
Quoted 2 lines
The code will continue to retry probe again and again until success only for plugged-in pci device matched the netvsc device.
Hi Matan,
The original code keeps writing to pipe even it's the same PCI device.
Yes, the
vdev_netvscwrites any plugged-in device to the associated netvsc device fd.Review
Quoted 7 lines
The
new code writes to pipe for a new device, only once. See the following code:(quoted diff) +3
-0+ /* Skip if this is same device already sent to failsafe */ + if (strcmp(addr, ctx->yield) == 0) + return 0;I understand you want to optimize the pipe writing to be written only after plugged-in hot event.
The current solution suffers from race: the PCI device may be plugged-out and plugged-in in short time shorter than the driver alarm delay, then the PCI device plugged-in detection will lost.
My suggestion:
Add validation to the plugged-in device probing state and that it is owned by failsafe(using ownership API) - don't write the pipe if so.Matan
Quoted 20 lines
This patch also saves lots of CPU since it no longer writes to pipe all the time. You are correct about the code will continue to probe on a new PCI device. But someone has to do it to handle hot-add.
Thanks,
LongQuoted 12 lines
Quoted 2 lines
It looks like device removal and add was never tested.
This is basic test we have to test plug-in plug-out and it passed every day in the last years.
Maybe something new and special in your setup?
Quoted 4 lines
If you test removal you will see that
vdev_netvsc:
1. Sends same PCI device repeatedly to failsafe (every alarm call) This is harmless, but useless.
2. When device is removed, keeps doing #1
Hi Thomas,
This issue has already been fixed, see:
http://mails.dpdk.org/archives/dev/2020-October/185921.html
It has been integrated, Long was able to test it and confirm it fixed this issue.
Quoted 153 lines · Thomas Monjalon
Fixing Gaetan's address
20/10/2020 00:33, Thomas Monjalon:
Quoted 145 lines
Gaetan, Matan,
Please could you check?25/09/2020 22:30, Long Li:
Quoted 137 lines
HI Matan,
While troubleshooting a failure in DPDK on device removal when VF device briefly disappears and comes back, I notice the failsafe driver is trying repeatedly to start a sub device (after this sub device has been successfully configured, but later hot removed from the kernel). This is due to repeated alarms calling
fs_dev_start(). I trace into this commit:ae80146 net/failsafe: fix removed device handling
The implementation of
fs_err()is interesting:(quoted diff) +7
-0+fs_err(struct sub_device *sdev, int err) +{ + /* A device removal shouldn't be reported as an error. */ + if (sdev->remove == 1 || err == -EIO) + return rte_errno = 0; + return err; +}If I change this function to:
(quoted diff) +1
-1@@ -497,7 +497,7 @@ int failsafe_eth_new_event_callback(uint16_t port_id fs_err(struct sub_device *sdev, int err) { /* A device removal shouldn't be reported as an error. */ - if (sdev->remove == 1 || err == -EIO) + if (sdev->remove == 1 && err == -EIO) return rte_errno = 0; return err; }The hung is going away. I don't know the reason why we use a || in the
if(). If a call torte_eth_dev_start()returning EIO (as the case infs_dev_start), the best choice would be bail out and fail this sub device.Can you please take a look?
Thanks,
Long________________________________
From: Matan Azrad <[email protected]>
Sent: Tuesday, September 15, 2020 12:00 AM
To: Long Li <[email protected]>; Stephen Hemminger <[email protected]>
Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Raslan Darawsheh <[email protected]>
Subject: RE: [dpdk-dev] [PATCH]net/vdev_netvsc: handle removal of associated pci deviceHi Li
From: Long Li <[email protected]>
Quoted 44 lines
Quoted 44 lines
Subject: Re: [dpdk-dev] [PATCH]
net/vdev_netvsc: handle removal of associated pci deviceHi Stephen
From: Stephen Hemminger:
Quoted 17 lines
Quoted 14 lines · On Sun, 6 Sep 2020 12:38:18 +0000 Matan Azrad <[email protected]> wrote:
Hi Stephen
From: Stephen Hemminger:
Quoted 10 lines
The
vdev_netvscwas not detecting when the associated PCI device (SRIOV) was removed. Because of that it would keep feeding the same
(removed) device to failsafe PMD which would then unsuccessfully try and probe for it.Change to use a mark/sweep method to detect that PCI device was removed, and also only tell failsafe about new PCI devices.
Vdev_netvscdoes not have to keep stuffing the pipe with the same already existing PCI device.Quoted 7 lines
Quoted 2 lines
As I know, the
vdev_netvscdriver doesn't call to failsafe if the PCI device isnot detected by the readlink command(considered as removed)...
Quoted 1 line
Am I missing something?
The original code is broken because
ctx_yieldis not cleared, it keeps sending the same value.Looking on the code again, It looks like
ctx->yieldhas no effect on the next pipe write, It is just used for log.After the PCI interface matching to the netvsc interface, the pipe write is triggered only if the readlink commands success to see the plugged-in PCI
device:
readlink /sys/class/net/[iface]/device/subsystem shows "pci" readlink /sys/class/net/[iface]/device shows the pci device ID.So, the assumption is when the above readlink failed on the interface the device is removed(plugged-out) and the fd write will not happen.
Quoted 6 lines
Quoted 2 lines
The code will continue to retry probe again and again until success only for plugged-in pci device matched the netvsc device.
Hi Matan,
The original code keeps writing to pipe even it's the same PCI device.
Yes, the
vdev_netvscwrites any plugged-in device to the associated netvsc device fd.Review
Quoted 7 lines
The
new code writes to pipe for a new device, only once. See the following code:(quoted diff) +3
-0+ /* Skip if this is same device already sent to failsafe */ + if (strcmp(addr, ctx->yield) == 0) + return 0;I understand you want to optimize the pipe writing to be written only after plugged-in hot event.
The current solution suffers from race: the PCI device may be plugged-out and plugged-in in short time shorter than the driver alarm delay, then the PCI device plugged-in detection will lost.
My suggestion:
Add validation to the plugged-in device probing state and that it is owned by failsafe(using ownership API) - don't write the pipe if so.Matan
Quoted 20 lines
This patch also saves lots of CPU since it no longer writes to pipe all the time. You are correct about the code will continue to probe on a new PCI device. But someone has to do it to handle hot-add.
Thanks,
LongQuoted 12 lines
Quoted 2 lines
It looks like device removal and add was never tested.
This is basic test we have to test plug-in plug-out and it passed every day in the last years.
Maybe something new and special in your setup?
Quoted 4 lines
If you test removal you will see that
vdev_netvsc:
1. Sends same PCI device repeatedly to failsafe (every alarm call) This is harmless, but useless.
2. When device is removed, keeps doing #1
Signature · 1 line
Gaëtan