RDMA/mana_ib: add support of multiple ports
Subsystems (2)
PATCH 1/1 RDMA/mana_ib: add support of multiple ports
From: Konstantin Taranov <[email protected]>
If the HW indicates support of multiple ports for rdma, create an IB device with a port per netdev in the ethernet mana driver.
CM is only available on port 1, but RC QPs are supported on all ports.
Signed-off-by Konstantin Taranov <[email protected]>
--- drivers/infiniband/hw/mana/device.c | 107 +++++++++++++-------------- drivers/infiniband/hw/mana/main.c | 13 +++- drivers/infiniband/hw/mana/mana_ib.h | 1 + 3 files changed, 64 insertions(+), 57 deletions(-)
drivers/infiniband/hw/mana/device.c +53 -54
@@ -77,28 +77,30 @@ static int mana_ib_netdev_event(struct notifier_block *this,struct gdma_context *gc = dev->gdma_dev->gdma_context;struct mana_context *mc = gc->mana.driver_data;struct net_device *ndev; +int i;/* Only process events from our parent device */ -if (event_dev != mc->ports[0 ]) -return NOTIFY_DONE ; - -switch (event) { -case NETDEV_CHANGEUPPER : - ndev =mana_get_primary_netdev (mc,0 , &dev->dev_tracker); -/* -* RDMA core will setup GID based on updated netdev. -* It's not possible to race with the core as rtnl lock is being -* held. -*/ -ib_device_set_netdev (&dev->ib_dev, ndev,1 ); - -/* mana_get_primary_netdev() returns ndev with refcount held */ -netdev_put (ndev, &dev->dev_tracker); - -return NOTIFY_OK ; -default : -return NOTIFY_DONE ; - } +for (i =0 ; i < dev->ib_dev.phys_port_cnt; i++) +if (event_dev == mc->ports[i]) { +switch (event) { +case NETDEV_CHANGEUPPER : + ndev =mana_get_primary_netdev (mc, i, &dev->dev_tracker); +/* +* RDMA core will setup GID based on updated netdev. +* It's not possible to race with the core as rtnl lock is being +* held. +*/ +ib_device_set_netdev (&dev->ib_dev, ndev, i +1 ); + +/* mana_get_primary_netdev() returns ndev with refcount held */ +netdev_put (ndev, &dev->dev_tracker); + +return NOTIFY_OK ; +default : +return NOTIFY_DONE ; + } + } +return NOTIFY_DONE ; } static int mana_ib_probe(struct auxiliary_device *adev,
@@ -111,7 +113,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,struct net_device *ndev;struct mana_ib_dev *dev;u8 mac_addr[ETH_ALEN ]; -int ret; +int ret, i; dev =ib_alloc_device (mana_ib_dev, ib_dev);if (!dev)
@@ -126,34 +128,11 @@ static int mana_ib_probe(struct auxiliary_device *adev,if (mana_ib_is_rnic (dev)) { dev->ib_dev.phys_port_cnt =1 ; - ndev =mana_get_primary_netdev (mc,0 , &dev->dev_tracker); -if (!ndev) { - ret = -ENODEV ; -ibdev_err (&dev->ib_dev,"Failed to get netdev for IB port 1" ); - goto free_ib_device; - } -ether_addr_copy (mac_addr, ndev->dev_addr); -addrconf_addr_eui48 ((u8 *)&dev->ib_dev.node_guid, ndev->dev_addr); - ret =ib_device_set_netdev (&dev->ib_dev, ndev,1 ); -/* mana_get_primary_netdev() returns ndev with refcount held */ -netdev_put (ndev, &dev->dev_tracker); -if (ret) { -ibdev_err (&dev->ib_dev,"Failed to set ib netdev, ret %d" , ret); - goto free_ib_device; - } - - dev->nb.notifier_call = mana_ib_netdev_event; - ret =register_netdevice_notifier (&dev->nb); -if (ret) { -ibdev_err (&dev->ib_dev,"Failed to register net notifier, %d" , - ret); - goto free_ib_device; - } - +addrconf_addr_eui48 ((u8 *)&dev->ib_dev.node_guid, mc->ports[0 ]->dev_addr); ret =mana_ib_gd_query_adapter_caps (dev);if (ret) {ibdev_err (&dev->ib_dev,"Failed to query device caps, ret %d" , ret); - goto deregister_net_notifier; + goto free_ib_device; }ib_set_device_ops (&dev->ib_dev, &mana_ib_stats_ops);
@@ -163,16 +142,36 @@ static int mana_ib_probe(struct auxiliary_device *adev, ret =mana_ib_create_eqs (dev);if (ret) {ibdev_err (&dev->ib_dev,"Failed to create EQs, ret %d" , ret); - goto deregister_net_notifier; + goto free_ib_device; } ret =mana_ib_gd_create_rnic_adapter (dev);if (ret) goto destroy_eqs; - ret =mana_ib_gd_config_mac (dev,ADDR_OP_ADD , mac_addr); +if (dev->adapter_caps.feature_flags &MANA_IB_FEATURE_MULTI_PORTS_SUPPORT ) + dev->ib_dev.phys_port_cnt = mc->num_ports; + +for (i =0 ; i < dev->ib_dev.phys_port_cnt; i++) { + ndev =mana_get_primary_netdev (mc, i, &dev->dev_tracker); +ether_addr_copy (mac_addr, ndev->dev_addr); + ret =ib_device_set_netdev (&dev->ib_dev, ndev, i +1 ); +/* mana_get_primary_netdev() returns ndev with refcount held */ +netdev_put (ndev, &dev->dev_tracker); +if (ret) { +ibdev_err (&dev->ib_dev,"Failed to set ib netdev, ret %d" , ret); + goto destroy_rnic; + } + ret =mana_ib_gd_config_mac (dev,ADDR_OP_ADD , mac_addr); +if (ret) { +ibdev_err (&dev->ib_dev,"Failed to add Mac address, ret %d" , ret); + goto destroy_rnic; + } + } + dev->nb.notifier_call = mana_ib_netdev_event; + ret =register_netdevice_notifier (&dev->nb);if (ret) { -ibdev_err (&dev->ib_dev,"Failed to add Mac address, ret %d" , ret); +ibdev_err (&dev->ib_dev,"Failed to register net notifier, %d" , ret); goto destroy_rnic; } }else {
@@ -188,7 +187,7 @@ static int mana_ib_probe(struct auxiliary_device *adev, MANA_AV_BUFFER_SIZE, 0);if (!dev->av_pool) { ret = -ENOMEM ; - goto destroy_rnic; + goto deregister_net_notifier; } ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,
@@ -205,15 +204,15 @@ static int mana_ib_probe(struct auxiliary_device *adev, deallocate_pool:dma_pool_destroy (dev->av_pool); +deregister_net_notifier: +if (mana_ib_is_rnic (dev)) +unregister_netdevice_notifier (&dev->nb); destroy_rnic:if (mana_ib_is_rnic (dev))mana_ib_gd_destroy_rnic_adapter (dev); destroy_eqs:if (mana_ib_is_rnic (dev))mana_ib_destroy_eqs (dev); -deregister_net_notifier: -if (mana_ib_is_rnic (dev)) -unregister_netdevice_notifier (&dev->nb); free_ib_device:xa_destroy (&dev->qp_table_wq);ib_dealloc_device (&dev->ib_dev);
@@ -227,9 +226,9 @@ static void mana_ib_remove(struct auxiliary_device *adev)ib_unregister_device (&dev->ib_dev);dma_pool_destroy (dev->av_pool);if (mana_ib_is_rnic (dev)) { +unregister_netdevice_notifier (&dev->nb);mana_ib_gd_destroy_rnic_adapter (dev);mana_ib_destroy_eqs (dev); -unregister_netdevice_notifier (&dev->nb); }xa_destroy (&dev->qp_table_wq);ib_dealloc_device (&dev->ib_dev);
drivers/infiniband/hw/mana/main.c +10 -3
@@ -563,8 +563,14 @@ int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num, immutable->gid_tbl_len = attr.gid_tbl_len;if (mana_ib_is_rnic (dev)) { - immutable->core_cap_flags =RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP ; - immutable->max_mad_size =IB_MGMT_MAD_SIZE ; +if (port_num ==1 ) { + immutable->core_cap_flags =RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP ; + immutable->max_mad_size =IB_MGMT_MAD_SIZE ; + }else { + immutable->core_cap_flags =RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP + |RDMA_CORE_CAP_ETH_AH ; + immutable->max_mad_size =0 ; + } }else { immutable->core_cap_flags =RDMA_CORE_PORT_RAW_PACKET ; }
@@ -633,8 +639,9 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port, props->pkey_tbl_len =1 ;if (mana_ib_is_rnic (dev)) { props->gid_tbl_len =16 ; - props->port_cap_flags =IB_PORT_CM_SUP ; props->ip_gids = true; +if (port ==1 ) + props->port_cap_flags =IB_PORT_CM_SUP ; }return 0 ;
drivers/infiniband/hw/mana/mana_ib.h +1 -0
@@ -220,6 +220,7 @@ struct mana_ib_query_adapter_caps_req {enum mana_ib_adapter_features {MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT =BIT (4 ),MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT =BIT (5 ), +MANA_IB_FEATURE_MULTI_PORTS_SUPPORT =BIT (6 ), };struct mana_ib_query_adapter_caps_resp {
Signature · 1 line
2.43.0
Quoted 8 lines · Konstantin Taranov
From: Konstantin Taranov <[email protected]>
If the HW indicates support of multiple ports for rdma, create an IB device with a port per netdev in the ethernet mana driver.
CM is only available on port 1, but RC QPs are supported on all ports.
Quoted 6 lines
Signed-off-by Konstantin Taranov <[email protected]>--- drivers/infiniband/hw/mana/device.c | 107 +++++++++++++-------------- drivers/infiniband/hw/mana/main.c | 13 +++- drivers/infiniband/hw/mana/mana_ib.h | 1 + 3 files changed, 64 insertions(+), 57 deletions(-)
mana_get_primary_netdev() can return NULL.
You should check return value even it can't be happen or remove "if (port_index >= ac->num_ports)" that returns NULL.
Thanks