net/mlx5: fix link state detection
cover net/mlx5: fix link state detection
This patchset fixes two related issues:
* In rare occasions with any HW link state change to UP was missed.
* If a port was DOWN before startup, its netdev would come UP, but appear DOWN in DPDK (especially probable with ConnectX-4).
Dmitry Kozlyuk (3):
common/mlx5: add Netlink event helpers
net/mlx5: fix link status change detection
net/mlx5: fix initial link status detection
drivers/common/mlx5/linux/mlx5_common_os.c | 2 +- drivers/common/mlx5/linux/mlx5_nl.c | 102 ++++++++++++++++++++- drivers/common/mlx5/linux/mlx5_nl.h | 8 +- drivers/common/mlx5/version.map | 2 + drivers/net/mlx5/linux/mlx5_ethdev_os.c | 63 ++++++++++--- drivers/net/mlx5/linux/mlx5_os.c | 72 +++++++++++++-- drivers/net/mlx5/linux/mlx5_vlan_os.c | 2 +- drivers/net/mlx5/mlx5.c | 1 + drivers/net/mlx5/mlx5.h | 3 + drivers/net/mlx5/mlx5_trigger.c | 12 ++- 10 files changed, 239 insertions(+), 28 deletions(-)
Signature · 1 line
2.25.1
PATCH 1/3 common/mlx5: add Netlink event helpers
Introduce mlx5_nl_read_events() to read Netlink events (technically, messages) from a socket that was configured to listen for them via a new mlx5_nl_init() parameter. Add mlx5_nl_parse_link_status_update() helper
to extract information from link-related events. This patch is a shared base for later fixes.
--- drivers/common/mlx5/linux/mlx5_common_os.c | 2 +- drivers/common/mlx5/linux/mlx5_nl.c | 102 ++++++++++++++++++++- drivers/common/mlx5/linux/mlx5_nl.h | 8 +- drivers/common/mlx5/version.map | 2 + drivers/net/mlx5/linux/mlx5_os.c | 8 +- drivers/net/mlx5/linux/mlx5_vlan_os.c | 2 +- 6 files changed, 116 insertions(+), 8 deletions(-)
drivers/common/mlx5/linux/mlx5_common_os.c +1 -1
@@ -487,7 +487,7 @@ mlx5_os_get_ibv_device(const struct rte_pci_addr *addr)static int mlx5_nl_roce_disable (const char *addr) { -int nlsk_fd =mlx5_nl_init (NETLINK_GENERIC ); +int nlsk_fd =mlx5_nl_init (NETLINK_GENERIC ,0 );int devlink_id;int enable;int ret;
drivers/common/mlx5/linux/mlx5_nl.c +101 -1
123 lines not on this page · raw message
drivers/common/mlx5/linux/mlx5_nl.h +7 -1
@@ -11,6 +11,7 @@#include "mlx5_common.h" +typedef void (mlx5_nl_event_cb )(struct nlmsghdr *hdr,void *user_data);/* VLAN netdev for VLAN workaround. */ struct mlx5_nl_vlan_dev {
@@ -30,7 +31,7 @@ struct mlx5_nl_vlan_vmwa_context { };__rte_internal -int mlx5_nl_init(int protocol); +int mlx5_nl_init(int protocol, int groups);__rte_internal int mlx5_nl_mac_addr_add(int nlsk_fd, unsigned int iface_idx, uint64_t *mac_own,struct rte_ether_addr *mac,uint32_t index);
@@ -75,4 +76,9 @@ int mlx5_nl_enable_roce_get(int nlsk_fd, int family_id, const char *pci_addr,int mlx5_nl_enable_roce_set (int nlsk_fd,int family_id,const char *pci_addr,int enable); +__rte_internal +int mlx5_nl_read_events(int nlsk_fd, mlx5_nl_event_cb *cb, void *cb_arg); +__rte_internal +int mlx5_nl_parse_link_status_update(struct nlmsghdr *hdr, uint32_t *ifindex); +#endif /* RTE_PMD_MLX5_NL_H_ */
drivers/common/mlx5/version.map +2 -0
@@ -123,9 +123,11 @@ INTERNAL { mlx5_nl_mac_addr_flush; # WINDOWS_NO_EXPORT mlx5_nl_mac_addr_remove; # WINDOWS_NO_EXPORT mlx5_nl_mac_addr_sync; # WINDOWS_NO_EXPORT + mlx5_nl_parse_link_status_update; # WINDOWS_NO_EXPORT mlx5_nl_port_state; # WINDOWS_NO_EXPORT mlx5_nl_portnum; # WINDOWS_NO_EXPORT mlx5_nl_promisc; # WINDOWS_NO_EXPORT + mlx5_nl_read_events; # WINDOWS_NO_EXPORT mlx5_nl_switch_info; # WINDOWS_NO_EXPORT mlx5_nl_vf_mac_addr_modify; # WINDOWS_NO_EXPORT mlx5_nl_vlan_vmwa_create; # WINDOWS_NO_EXPORT
drivers/net/mlx5/linux/mlx5_os.c +4 -4
@@ -1086,7 +1086,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, " old OFED/rdma-core version or firmware configuration");#endif config->mpls_en = mpls_en; - nl_rdma =mlx5_nl_init (NETLINK_RDMA ); + nl_rdma =mlx5_nl_init (NETLINK_RDMA ,0 );/* Check port status. */ if (spawn->phys_port <=UINT8_MAX ) {/* Legacy Verbs api only support u8 port number. */
@@ -1133,7 +1133,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, priv->mtu =RTE_ETHER_MTU ;/* Some internal functions rely on Netlink sockets, open them now. */ priv->nl_socket_rdma = nl_rdma; - priv->nl_socket_route =mlx5_nl_init (NETLINK_ROUTE ); + priv->nl_socket_route =mlx5_nl_init (NETLINK_ROUTE ,0 ); priv->representor = !!switch_info->representor; priv->master = !!switch_info->master; priv->domain_id =RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID ;
@@ -2130,8 +2130,8 @@ mlx5_os_pci_probe_pf(struct mlx5_common_device *cdev,* matching ones, gathering into the list. */ struct ibv_device *ibv_match[ret +1 ]; -int nl_route =mlx5_nl_init (NETLINK_ROUTE ); -int nl_rdma =mlx5_nl_init (NETLINK_RDMA ); +int nl_route =mlx5_nl_init (NETLINK_ROUTE ,0 ); +int nl_rdma =mlx5_nl_init (NETLINK_RDMA ,0 );unsigned int i;while (ret-- >0 ) {
drivers/net/mlx5/linux/mlx5_vlan_os.c +1 -1
@@ -136,7 +136,7 @@ mlx5_vlan_vmwa_init(struct rte_eth_dev *dev, uint32_t ifindex)return NULL ; }rte_spinlock_init (&vmwa->sl); - vmwa->nl_socket =mlx5_nl_init (NETLINK_ROUTE ); + vmwa->nl_socket =mlx5_nl_init (NETLINK_ROUTE ,0 );if (vmwa->nl_socket <0 ) {DRV_LOG (WARNING , "Can not create Netlink socket"
Signature · 1 line
2.25.1
PATCH 2/3 net/mlx5: fix link status change detection
In reply to Dmitry KozlyukSometimes net/mlx5 devices did not detect link status change to "up".
Each shared device was monitoring IBV_EVENT_PORT_{ACTIVE,ERR} and queried the link status upon receiving the event. IBV_EVENT_PORT_ACTIVE is delivered when the logical link status (UP flag) is set, but the physical link status (RUNNING flag) may be down at that time, in which case the new link status would be erroneously considered down.
IBV interface is insufficient for the task.
Monitor interface events using Netlink.
---
Coverity complains about unchecked rte_intr_fd/type_set() return values. That would be useless, because rte_intr_handle is in a known state.
drivers/net/mlx5/linux/mlx5_ethdev_os.c | 63 ++++++++++++++++++++----- drivers/net/mlx5/linux/mlx5_os.c | 55 +++++++++++++++++++++ drivers/net/mlx5/mlx5.c | 1 + drivers/net/mlx5/mlx5.h | 3 ++ drivers/net/mlx5/mlx5_trigger.c | 12 ++++- 5 files changed, 120 insertions(+), 14 deletions(-)
drivers/net/mlx5/linux/mlx5_ethdev_os.c +51 -12
@@ -38,6 +38,7 @@#include <mlx5_devx_cmds.h> #include <mlx5_common.h> #include <mlx5_malloc.h> +#include <mlx5_nl.h> #include "mlx5.h" #include "mlx5_rxtx.h"
@@ -760,6 +761,56 @@ mlx5_dev_interrupt_device_fatal(struct mlx5_dev_ctx_shared *sh) } } +static void +mlx5_dev_interrupt_nl_cb (struct nlmsghdr *hdr,void *cb_arg) +{ +struct mlx5_dev_ctx_shared *sh = cb_arg; +uint32_t i; +uint32_t if_index; + +if (mlx5_nl_parse_link_status_update (hdr, &if_index) <0 ) +return ; +for (i =0 ; i < sh->max_port; i++) { +struct mlx5_dev_shared_port *port = &sh->port[i]; +struct rte_eth_dev *dev; +struct mlx5_priv *priv; + +if (port->nl_ih_port_id >=RTE_MAX_ETHPORTS ) +continue ; + dev = &rte_eth_devices[port->nl_ih_port_id]; +/* Probing may initiate an LSC before configuration is done. */ +if (dev->data->dev_configured && + !dev->data->dev_conf.intr_conf.lsc) +break ; + priv = dev->data->dev_private; +if (priv->if_index == if_index) { +/* Block logical LSC events. */ +uint16_t prev_status = dev->data->dev_link.link_status; + +if (mlx5_link_update (dev,0 ) <0 ) +DRV_LOG (ERR ,"Failed to update link status: %s" , +rte_strerror (rte_errno)); +else if (prev_status != dev->data->dev_link.link_status) +rte_eth_dev_callback_process + (dev,RTE_ETH_EVENT_INTR_LSC ,NULL ); +break ; + } + } +} + +void +mlx5_dev_interrupt_handler_nl (void *arg) +{ +struct mlx5_dev_ctx_shared *sh = arg; +int nlsk_fd =rte_intr_fd_get (sh->intr_handle_nl); + +if (nlsk_fd <0 ) +return ; +if (mlx5_nl_read_events (nlsk_fd, mlx5_dev_interrupt_nl_cb, sh) <0 ) +DRV_LOG (ERR ,"Failed to process Netlink events: %s" , +rte_strerror (rte_errno)); +} +/** * Handle shared asynchronous events the NIC (removal event * and link status change). Supports multiport IB device.
@@ -823,18 +874,6 @@ mlx5_dev_interrupt_handler(void *cb_arg) tmp = sh->port[tmp -1 ].ih_port_id; dev = &rte_eth_devices[tmp];MLX5_ASSERT (dev); -if ((event.event_type ==IBV_EVENT_PORT_ACTIVE || - event.event_type ==IBV_EVENT_PORT_ERR ) && - dev->data->dev_conf.intr_conf.lsc) { - mlx5_glue->ack_async_event (&event); -if (mlx5_link_update (dev,0 ) == -EAGAIN ) { -usleep (0 ); -continue ; - } -rte_eth_dev_callback_process - (dev,RTE_ETH_EVENT_INTR_LSC ,NULL ); -continue ; - }DRV_LOG (DEBUG ,"port %u cannot handle an unknown event (type %d)" , dev->data->port_id, event.event_type);
drivers/net/mlx5/linux/mlx5_os.c +55 -0
77 lines not on this page · raw message
drivers/net/mlx5/mlx5.c +1 -0
@@ -1206,6 +1206,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,for (i =0 ; i < sh->max_port; i++) { sh->port[i].ih_port_id =RTE_MAX_ETHPORTS ; sh->port[i].devx_ih_port_id =RTE_MAX_ETHPORTS ; + sh->port[i].nl_ih_port_id =RTE_MAX_ETHPORTS ; }if (sh->devx) { sh->td =mlx5_devx_cmd_create_td (sh->cdev->ctx);
drivers/net/mlx5/mlx5.h +3 -0
@@ -605,6 +605,7 @@ struct mlx5_age_info {struct mlx5_dev_shared_port {uint32_t ih_port_id;uint32_t devx_ih_port_id; +uint32_t nl_ih_port_id;/* * Interrupt handler port_id. Used by shared interrupt * handler to find the corresponding rte_eth device
@@ -1203,6 +1204,7 @@ struct mlx5_dev_ctx_shared {/* Shared interrupt handler section. */ struct rte_intr_handle *intr_handle;/* Interrupt handler for device. */ struct rte_intr_handle *intr_handle_devx;/* DEVX interrupt handler. */ +struct rte_intr_handle *intr_handle_nl;/* Netlink interrupt handler. */ void *devx_comp;/* DEVX async comp obj. */ struct mlx5_devx_obj *tis[16 ];/* TIS object. */ struct mlx5_devx_obj *td;/* Transport domain. */
@@ -1585,6 +1587,7 @@ int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf);void mlx5_dev_interrupt_handler (void *arg);void mlx5_dev_interrupt_handler_devx (void *arg); +void mlx5_dev_interrupt_handler_nl (void *arg);int mlx5_set_link_down (struct rte_eth_dev *dev);int mlx5_set_link_up (struct rte_eth_dev *dev);int mlx5_is_removed (struct rte_eth_dev *dev);
drivers/net/mlx5/mlx5_trigger.c +10 -2
@@ -1205,11 +1205,18 @@ mlx5_dev_start(struct rte_eth_dev *dev) priv->sh->port[priv->dev_port -1 ].ih_port_id = (uint32_t )dev->data->port_id; }else { -DRV_LOG (INFO ,"port %u starts without LSC and RMV interrupts." , +DRV_LOG (INFO ,"port %u starts without RMV interrupts." , dev->data->port_id); - dev->data->dev_conf.intr_conf.lsc =0 ; dev->data->dev_conf.intr_conf.rmv =0 ; } +if (rte_intr_fd_get (priv->sh->intr_handle_nl) >=0 ) { + priv->sh->port[priv->dev_port -1 ].nl_ih_port_id = + (uint32_t )dev->data->port_id; + }else { +DRV_LOG (INFO ,"port %u starts without LSC interrupts." , + dev->data->port_id); + dev->data->dev_conf.intr_conf.lsc =0 ; + }if (rte_intr_fd_get (priv->sh->intr_handle_devx) >=0 ) priv->sh->port[priv->dev_port -1 ].devx_ih_port_id = (uint32_t )dev->data->port_id;
@@ -1261,6 +1268,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)mlx5_rx_intr_vec_disable (dev); priv->sh->port[priv->dev_port -1 ].ih_port_id =RTE_MAX_ETHPORTS ; priv->sh->port[priv->dev_port -1 ].devx_ih_port_id =RTE_MAX_ETHPORTS ; + priv->sh->port[priv->dev_port -1 ].nl_ih_port_id =RTE_MAX_ETHPORTS ;mlx5_txq_stop (dev);mlx5_rxq_stop (dev);if (priv->obj_ops.lb_dummy_queue_release)
Signature · 1 line
2.25.1
PATCH 3/3 net/mlx5: fix initial link status detection
In reply to Dmitry KozlyukLink status change takes time that depends on the HW and the kernel. It was checked immediately after the change was issued at probing. If the port had beed down before probing, a "down" state may be read, while the port would be "up" imminently.
After that, DPDK reported the port as "down" mistakenly and "ifconfig $DEV up" did not trigger an LSC event, because from the system's perspective the port was "up" already.
Install Netlink event handler at port probe before requesting the port to come up in order to receive LSC event even if it comes up between probe and start.
--- drivers/net/mlx5/linux/mlx5_os.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +3 -6
@@ -1645,13 +1645,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,/* Bring Ethernet device up. */ DRV_LOG (DEBUG ,"port %u forcing Ethernet interface up" , eth_dev->data->port_id); +/* Watch LSC interrupts between port probe and port start. */ + priv->sh->port[priv->dev_port -1 ].nl_ih_port_id = + eth_dev->data->port_id;mlx5_set_link_up (eth_dev); -/* -* Even though the interrupt handler is not installed yet, -* interrupts will still trigger on the async_fd from -* Verbs context returned by ibv_open_device(). -*/ -mlx5_link_update (eth_dev,0 );#ifdef HAVE_MLX5DV_DR_ESWITCH if (!(config->hca_attr.eswitch_manager && config->dv_flow_en && (switch_info->representor || switch_info->master)))
Signature · 1 line
2.25.1
v2 cover net/mlx5: fix link state detection
In reply to Dmitry KozlyukThis patchset fixes two related issues:
* In rare occasions with any HW link state change to UP was missed.
* If a port was DOWN before startup, its netdev would come UP, but appear DOWN in DPDK (especially probable with ConnectX-4).
v2: fix not detecting the link up when it was up before the app start.
Dmitry Kozlyuk (3):
common/mlx5: add Netlink event helpers
net/mlx5: fix link status change detection
net/mlx5: fix initial link status detection
drivers/common/mlx5/linux/mlx5_common_os.c | 2 +- drivers/common/mlx5/linux/mlx5_nl.c | 102 ++++++++++++++++++++- drivers/common/mlx5/linux/mlx5_nl.h | 8 +- drivers/common/mlx5/version.map | 2 + drivers/net/mlx5/linux/mlx5_ethdev_os.c | 63 ++++++++++--- drivers/net/mlx5/linux/mlx5_os.c | 74 +++++++++++++-- drivers/net/mlx5/linux/mlx5_vlan_os.c | 2 +- drivers/net/mlx5/mlx5.c | 1 + drivers/net/mlx5/mlx5.h | 3 + drivers/net/mlx5/mlx5_trigger.c | 12 ++- 10 files changed, 241 insertions(+), 28 deletions(-)
Signature · 1 line
2.25.1
PATCH v2 1/3 common/mlx5: add Netlink event helpers
Introduce mlx5_nl_read_events() to read Netlink events (technically, messages) from a socket that was configured to listen for them via a new mlx5_nl_init() parameter. Add mlx5_nl_parse_link_status_update() helper
to extract information from link-related events. This patch is a shared base for later fixes.
--- drivers/common/mlx5/linux/mlx5_common_os.c | 2 +- drivers/common/mlx5/linux/mlx5_nl.c | 102 ++++++++++++++++++++- drivers/common/mlx5/linux/mlx5_nl.h | 8 +- drivers/common/mlx5/version.map | 2 + drivers/net/mlx5/linux/mlx5_os.c | 8 +- drivers/net/mlx5/linux/mlx5_vlan_os.c | 2 +- 6 files changed, 116 insertions(+), 8 deletions(-)
drivers/common/mlx5/linux/mlx5_common_os.c +1 -1
@@ -590,7 +590,7 @@ mlx5_os_get_ibv_device(const struct rte_pci_addr *addr)static int mlx5_nl_roce_disable (const char *addr) { -int nlsk_fd =mlx5_nl_init (NETLINK_GENERIC ); +int nlsk_fd =mlx5_nl_init (NETLINK_GENERIC ,0 );int devlink_id;int enable;int ret;
drivers/common/mlx5/linux/mlx5_nl.c +101 -1
123 lines not on this page · raw message
drivers/common/mlx5/linux/mlx5_nl.h +7 -1
@@ -11,6 +11,7 @@#include "mlx5_common.h" +typedef void (mlx5_nl_event_cb )(struct nlmsghdr *hdr,void *user_data);/* VLAN netdev for VLAN workaround. */ struct mlx5_nl_vlan_dev {
@@ -30,7 +31,7 @@ struct mlx5_nl_vlan_vmwa_context { };__rte_internal -int mlx5_nl_init(int protocol); +int mlx5_nl_init(int protocol, int groups);__rte_internal int mlx5_nl_mac_addr_add(int nlsk_fd, unsigned int iface_idx, uint64_t *mac_own,struct rte_ether_addr *mac,uint32_t index);
@@ -75,4 +76,9 @@ int mlx5_nl_enable_roce_get(int nlsk_fd, int family_id, const char *pci_addr,int mlx5_nl_enable_roce_set (int nlsk_fd,int family_id,const char *pci_addr,int enable); +__rte_internal +int mlx5_nl_read_events(int nlsk_fd, mlx5_nl_event_cb *cb, void *cb_arg); +__rte_internal +int mlx5_nl_parse_link_status_update(struct nlmsghdr *hdr, uint32_t *ifindex); +#endif /* RTE_PMD_MLX5_NL_H_ */
drivers/common/mlx5/version.map +2 -0
@@ -127,9 +127,11 @@ INTERNAL { mlx5_nl_mac_addr_flush; # WINDOWS_NO_EXPORT mlx5_nl_mac_addr_remove; # WINDOWS_NO_EXPORT mlx5_nl_mac_addr_sync; # WINDOWS_NO_EXPORT + mlx5_nl_parse_link_status_update; # WINDOWS_NO_EXPORT mlx5_nl_port_state; # WINDOWS_NO_EXPORT mlx5_nl_portnum; # WINDOWS_NO_EXPORT mlx5_nl_promisc; # WINDOWS_NO_EXPORT + mlx5_nl_read_events; # WINDOWS_NO_EXPORT mlx5_nl_switch_info; # WINDOWS_NO_EXPORT mlx5_nl_vf_mac_addr_modify; # WINDOWS_NO_EXPORT mlx5_nl_vlan_vmwa_create; # WINDOWS_NO_EXPORT
drivers/net/mlx5/linux/mlx5_os.c +4 -4
@@ -1117,7 +1117,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, sh =mlx5_alloc_shared_dev_ctx (spawn, mkvlist);if (!sh)return NULL ; - nl_rdma =mlx5_nl_init (NETLINK_RDMA ); + nl_rdma =mlx5_nl_init (NETLINK_RDMA ,0 );/* Check port status. */ if (spawn->phys_port <=UINT8_MAX ) {/* Legacy Verbs api only support u8 port number. */
@@ -1180,7 +1180,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, priv->mtu =RTE_ETHER_MTU ;/* Some internal functions rely on Netlink sockets, open them now. */ priv->nl_socket_rdma = nl_rdma; - priv->nl_socket_route =mlx5_nl_init (NETLINK_ROUTE ); + priv->nl_socket_route =mlx5_nl_init (NETLINK_ROUTE ,0 ); priv->representor = !!switch_info->representor; priv->master = !!switch_info->master; priv->domain_id =RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID ;
@@ -1927,8 +1927,8 @@ mlx5_os_pci_probe_pf(struct mlx5_common_device *cdev,* matching ones, gathering into the list. */ struct ibv_device *ibv_match[ret +1 ]; -int nl_route =mlx5_nl_init (NETLINK_ROUTE ); -int nl_rdma =mlx5_nl_init (NETLINK_RDMA ); +int nl_route =mlx5_nl_init (NETLINK_ROUTE ,0 ); +int nl_rdma =mlx5_nl_init (NETLINK_RDMA ,0 );unsigned int i;while (ret-- >0 ) {
drivers/net/mlx5/linux/mlx5_vlan_os.c +1 -1
@@ -135,7 +135,7 @@ mlx5_vlan_vmwa_init(struct rte_eth_dev *dev, uint32_t ifindex)return NULL ; }rte_spinlock_init (&vmwa->sl); - vmwa->nl_socket =mlx5_nl_init (NETLINK_ROUTE ); + vmwa->nl_socket =mlx5_nl_init (NETLINK_ROUTE ,0 );if (vmwa->nl_socket <0 ) {DRV_LOG (WARNING , "Can not create Netlink socket"
Signature · 1 line
2.25.1
Re: [PATCH v2 1/3] common/mlx5: add Netlink event helpers
Quoted 9 lines · Dmitry Kozlyuk
Introduce
mlx5_nl_read_events()to read Netlink events (technically, messages) from a socket that was configured to listen for them via a newmlx5_nl_init()parameter. Addmlx5_nl_parse_link_status_update()helper
to extract information from link-related events. This patch is a shared base for later fixes.
Hi Dmitry,
For clarification, this patch is not fix, but it is requested to be backported to be able to backport fixes in this patchset, right?
This looks OK to me, but cc'ed LTS maintainers in case they have objection.
Quoted 2 lines
Signed-off-by Dmitry Kozlyuk <[email protected]>Reviewed-by Viacheslav Ovsiienko <[email protected]>
<...>
RE: [PATCH v2 1/3] common/mlx5: add Netlink event helpers
Hi Ferruh,
Quoted 2 lines
-----Original Message-----
From: Ferruh Yigit <[email protected]>
[...]
Quoted 5 lines
Hi Dmitry,
For clarification, this patch is not fix, but it is requested to be backported to be able to backport fixes in this patchset, right?
Yes.
Re: [PATCH v2 1/3] common/mlx5: add Netlink event helpers
Quoted 4 lines · Dmitry Kozlyuk
Hi Ferruh,
Quoted 2 lines
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Quoted 8 lines
[...]
Quoted 5 lines
Hi Dmitry,
For clarification, this patch is not fix, but it is requested to be backported to be able to backport fixes in this patchset, right?
Yes.
The updated API is internal so that should be ok. I'm ok to take this on 21.11 as long as you can confirm it's not breaking any user compatibility with external sw versions/ABI/API etc from 21.11.0 ?
Assuming that's ok, please send a rebased series for 21.11. I'm not comfortable rebasing the series with the amount of changes on dpdk main branch to same functions in mlx5_os.c.
P.S. Better to rebase on patch queue [0] to avoid conflicts with other backports not pushed to dpdk.org yet.
thanks,
Kevin.
RE: [PATCH v2 1/3] common/mlx5: add Netlink event helpers
Hi Kevin,
Quoted 2 lines
-----Original Message-----
From: Kevin Traynor <[email protected]>
[...]
Quoted 8 lines
The updated API is internal so that should be ok. I'm ok to take this on
21.11 as long as you can confirm it's not breaking any user compatibility with external sw versions/ABI/API etc from 21.11.0 ?Assuming that's ok, please send a rebased series for 21.11. I'm not comfortable rebasing the series with the amount of changes on dpdk main branch to same functions in
mlx5_os.c.
Changes are not breaking any external SW.
Backport sent:
http://inbox.dpdk.org/stable/[email protected]
PATCH v2 2/3 net/mlx5: fix link status change detection
In reply to Dmitry KozlyukSometimes net/mlx5 devices did not detect link status change to "up".
Each shared device was monitoring IBV_EVENT_PORT_{ACTIVE,ERR} and queried the link status upon receiving the event. IBV_EVENT_PORT_ACTIVE is delivered when the logical link status (UP flag) is set, but the physical link status (RUNNING flag) may be down at that time, in which case the new link status would be erroneously considered down.
IBV interface is insufficient for the task.
Monitor interface events using Netlink.
---
Coverity complains about unchecked rte_intr_fd/type_set() return values. That would be useless, because rte_intr_handle is in a known state.
drivers/net/mlx5/linux/mlx5_ethdev_os.c | 63 ++++++++++++++++++++----- drivers/net/mlx5/linux/mlx5_os.c | 55 +++++++++++++++++++++ drivers/net/mlx5/mlx5.c | 1 + drivers/net/mlx5/mlx5.h | 3 ++ drivers/net/mlx5/mlx5_trigger.c | 12 ++++- 5 files changed, 120 insertions(+), 14 deletions(-)
drivers/net/mlx5/linux/mlx5_ethdev_os.c +51 -12
@@ -38,6 +38,7 @@#include <mlx5_devx_cmds.h> #include <mlx5_common.h> #include <mlx5_malloc.h> +#include <mlx5_nl.h> #include "mlx5.h" #include "mlx5_rxtx.h"
@@ -760,6 +761,56 @@ mlx5_dev_interrupt_device_fatal(struct mlx5_dev_ctx_shared *sh) } } +static void +mlx5_dev_interrupt_nl_cb (struct nlmsghdr *hdr,void *cb_arg) +{ +struct mlx5_dev_ctx_shared *sh = cb_arg; +uint32_t i; +uint32_t if_index; + +if (mlx5_nl_parse_link_status_update (hdr, &if_index) <0 ) +return ; +for (i =0 ; i < sh->max_port; i++) { +struct mlx5_dev_shared_port *port = &sh->port[i]; +struct rte_eth_dev *dev; +struct mlx5_priv *priv; + +if (port->nl_ih_port_id >=RTE_MAX_ETHPORTS ) +continue ; + dev = &rte_eth_devices[port->nl_ih_port_id]; +/* Probing may initiate an LSC before configuration is done. */ +if (dev->data->dev_configured && + !dev->data->dev_conf.intr_conf.lsc) +break ; + priv = dev->data->dev_private; +if (priv->if_index == if_index) { +/* Block logical LSC events. */ +uint16_t prev_status = dev->data->dev_link.link_status; + +if (mlx5_link_update (dev,0 ) <0 ) +DRV_LOG (ERR ,"Failed to update link status: %s" , +rte_strerror (rte_errno)); +else if (prev_status != dev->data->dev_link.link_status) +rte_eth_dev_callback_process + (dev,RTE_ETH_EVENT_INTR_LSC ,NULL ); +break ; + } + } +} + +void +mlx5_dev_interrupt_handler_nl (void *arg) +{ +struct mlx5_dev_ctx_shared *sh = arg; +int nlsk_fd =rte_intr_fd_get (sh->intr_handle_nl); + +if (nlsk_fd <0 ) +return ; +if (mlx5_nl_read_events (nlsk_fd, mlx5_dev_interrupt_nl_cb, sh) <0 ) +DRV_LOG (ERR ,"Failed to process Netlink events: %s" , +rte_strerror (rte_errno)); +} +/** * Handle shared asynchronous events the NIC (removal event * and link status change). Supports multiport IB device.
@@ -823,18 +874,6 @@ mlx5_dev_interrupt_handler(void *cb_arg) tmp = sh->port[tmp -1 ].ih_port_id; dev = &rte_eth_devices[tmp];MLX5_ASSERT (dev); -if ((event.event_type ==IBV_EVENT_PORT_ACTIVE || - event.event_type ==IBV_EVENT_PORT_ERR ) && - dev->data->dev_conf.intr_conf.lsc) { - mlx5_glue->ack_async_event (&event); -if (mlx5_link_update (dev,0 ) == -EAGAIN ) { -usleep (0 ); -continue ; - } -rte_eth_dev_callback_process - (dev,RTE_ETH_EVENT_INTR_LSC ,NULL ); -continue ; - }DRV_LOG (DEBUG ,"port %u cannot handle an unknown event (type %d)" , dev->data->port_id, event.event_type);
drivers/net/mlx5/linux/mlx5_os.c +55 -0
77 lines not on this page · raw message
drivers/net/mlx5/mlx5.c +1 -0
@@ -1457,6 +1457,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,for (i =0 ; i < sh->max_port; i++) { sh->port[i].ih_port_id =RTE_MAX_ETHPORTS ; sh->port[i].devx_ih_port_id =RTE_MAX_ETHPORTS ; + sh->port[i].nl_ih_port_id =RTE_MAX_ETHPORTS ; }if (sh->cdev->config.devx) { sh->td =mlx5_devx_cmd_create_td (sh->cdev->ctx);
drivers/net/mlx5/mlx5.h +3 -0
@@ -637,6 +637,7 @@ struct mlx5_age_info {struct mlx5_dev_shared_port {uint32_t ih_port_id;uint32_t devx_ih_port_id; +uint32_t nl_ih_port_id;/* * Interrupt handler port_id. Used by shared interrupt * handler to find the corresponding rte_eth device
@@ -1239,6 +1240,7 @@ struct mlx5_dev_ctx_shared {/* Shared interrupt handler section. */ struct rte_intr_handle *intr_handle;/* Interrupt handler for device. */ struct rte_intr_handle *intr_handle_devx;/* DEVX interrupt handler. */ +struct rte_intr_handle *intr_handle_nl;/* Netlink interrupt handler. */ void *devx_comp;/* DEVX async comp obj. */ struct mlx5_devx_obj *tis[16 ];/* TIS object. */ struct mlx5_devx_obj *td;/* Transport domain. */
@@ -1666,6 +1668,7 @@ int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf);void mlx5_dev_interrupt_handler (void *arg);void mlx5_dev_interrupt_handler_devx (void *arg); +void mlx5_dev_interrupt_handler_nl (void *arg);int mlx5_set_link_down (struct rte_eth_dev *dev);int mlx5_set_link_up (struct rte_eth_dev *dev);int mlx5_is_removed (struct rte_eth_dev *dev);
drivers/net/mlx5/mlx5_trigger.c +10 -2
@@ -1207,11 +1207,18 @@ mlx5_dev_start(struct rte_eth_dev *dev) priv->sh->port[priv->dev_port -1 ].ih_port_id = (uint32_t )dev->data->port_id; }else { -DRV_LOG (INFO ,"port %u starts without LSC and RMV interrupts." , +DRV_LOG (INFO ,"port %u starts without RMV interrupts." , dev->data->port_id); - dev->data->dev_conf.intr_conf.lsc =0 ; dev->data->dev_conf.intr_conf.rmv =0 ; } +if (rte_intr_fd_get (priv->sh->intr_handle_nl) >=0 ) { + priv->sh->port[priv->dev_port -1 ].nl_ih_port_id = + (uint32_t )dev->data->port_id; + }else { +DRV_LOG (INFO ,"port %u starts without LSC interrupts." , + dev->data->port_id); + dev->data->dev_conf.intr_conf.lsc =0 ; + }if (rte_intr_fd_get (priv->sh->intr_handle_devx) >=0 ) priv->sh->port[priv->dev_port -1 ].devx_ih_port_id = (uint32_t )dev->data->port_id;
@@ -1263,6 +1270,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)mlx5_rx_intr_vec_disable (dev); priv->sh->port[priv->dev_port -1 ].ih_port_id =RTE_MAX_ETHPORTS ; priv->sh->port[priv->dev_port -1 ].devx_ih_port_id =RTE_MAX_ETHPORTS ; + priv->sh->port[priv->dev_port -1 ].nl_ih_port_id =RTE_MAX_ETHPORTS ;mlx5_txq_stop (dev);mlx5_rxq_stop (dev);if (priv->obj_ops.lb_dummy_queue_release)
Signature · 1 line
2.25.1
PATCH v2 3/3 net/mlx5: fix initial link status detection
In reply to Dmitry KozlyukLink status change takes time that depends on the HW and the kernel. It was checked immediately after the change was issued at probing. If the port had beed down before probing, a "down" state may be read, while the port would be "up" imminently.
After that, DPDK reported the port as "down" mistakenly and "ifconfig $DEV up" did not trigger an LSC event, because from the system's perspective the port was "up" already.
Install Netlink event handler at port probe before requesting the port to come up in order to receive LSC event even if it comes up between probe and start.
--- drivers/net/mlx5/linux/mlx5_os.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +5 -6
@@ -1483,13 +1483,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,/* Bring Ethernet device up. */ DRV_LOG (DEBUG ,"port %u forcing Ethernet interface up" , eth_dev->data->port_id); -mlx5_set_link_up (eth_dev); -/* -* Even though the interrupt handler is not installed yet, -* interrupts will still trigger on the async_fd from -* Verbs context returned by ibv_open_device(). -*/ +/* Read link status in case it is up and there will be no event. */ mlx5_link_update (eth_dev,0 ); +/* Watch LSC interrupts between port probe and port start. */ + priv->sh->port[priv->dev_port -1 ].nl_ih_port_id = + eth_dev->data->port_id; +mlx5_set_link_up (eth_dev);for (i =0 ; i <MLX5_FLOW_TYPE_MAXI ; i++) { icfg[i].release_mem_en = !!sh->config.reclaim_mode;if (sh->config.reclaim_mode)
Signature · 1 line
2.25.1
Hi,
Quoted 30 lines
-----Original Message-----
From: Dmitry Kozlyuk <[email protected]>
Sent: Tuesday, March 1, 2022 2:15 PM
To: [email protected]
Subject: [PATCH v2 0/3] net/mlx5: fix link state detectionThis patchset fixes two related issues:
* In rare occasions with any HW link state change to UP was missed.
* If a port was DOWN before startup, its netdev would come UP, but appear DOWN in DPDK (especially probable with ConnectX-4).v2: fix not detecting the link up when it was up before the app start.
Dmitry Kozlyuk (3):
common/mlx5: add Netlink event helpers
net/mlx5: fix link status change detection
net/mlx5: fix initial link status detectiondrivers/common/mlx5/linux/mlx5_common_os.c | 2 +- drivers/common/mlx5/linux/mlx5_nl.c | 102 ++++++++++++++++++++- drivers/common/mlx5/linux/mlx5_nl.h | 8 +- drivers/common/mlx5/version.map | 2 + drivers/net/mlx5/linux/mlx5_ethdev_os.c | 63 ++++++++++--- drivers/net/mlx5/linux/mlx5_os.c | 74 +++++++++++++-- drivers/net/mlx5/linux/mlx5_vlan_os.c | 2 +- drivers/net/mlx5/mlx5.c | 1 + drivers/net/mlx5/mlx5.h | 3 + drivers/net/mlx5/mlx5_trigger.c | 12 ++- 10 files changed, 241 insertions(+), 28 deletions(-)
Quoted 2 lines
--
2.25.1
Series applied to next-net-mlx,
Kindest regards,
Raslan Darawsheh