RDMA/bnxt_re driver update
Subsystems (2)
Selvin Xavier
maintainerv3 cover RDMA/bnxt_re driver update
Includes code refactoring in the device init/deinit path and use the new driver unregistration APIs.
Please apply to for-next.
v2 -> v3:
- Droped the patch which was adding more state macros
- To prevent addition of any device during driver removal, unregister netdev notifier and delete the driver's workqueu before calling ib_unregister_driver
v1-> v2:
- Remove the patches 1,2 and 6 from the v1 series.
They are already merged.
- Added ASSERT_RTNL instead of comment in Patch 2
- For Patch 3, explicitly queue the removal of the VF devices before calling ib_unregister_driver. This can avoid command timeouts seen, if the PFs gets removed before the VFs. Previous discussion - https://patchwork.kernel.org/patch/11260013/
Selvin Xavier (2):
RDMA/bnxt_re: Refactor device add/remove functionalities RDMA/bnxt_re: Use driver_unregister and unregistration API
drivers/infiniband/hw/bnxt_re/main.c | 209 +++++++++++++++++------------------ 1 file changed, 104 insertions(+), 105 deletions(-)
Signature · 1 line
2.5.5
Selvin Xavier
maintainerPATCH v3 1/2 RDMA/bnxt_re: Refactor device add/remove functionalities
- bnxt_re_ib_reg() handles two main functionalities - initializing the device and registering with the IB stack. Split it into 2 functions i.e. bnxt_re_dev_init() and bnxt_re_ib_init() to account for the same thereby improve modularity. Do the same for bnxt_re_ib_unreg()i.e. split into two functions - bnxt_re_dev_uninit() and bnxt_re_ib_uninit().
- Simplify the code by combining the different steps to add and remove the device into two functions.
--- drivers/infiniband/hw/bnxt_re/main.c | 135 ++++++++++++++++++++--------------- 1 file changed, 78 insertions(+), 57 deletions(-)
drivers/infiniband/hw/bnxt_re/main.c +78 -57
@@ -78,7 +78,8 @@ static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list);/* Mutex to protect the list of bnxt_re devices added */ static DEFINE_MUTEX (bnxt_re_dev_lock);static struct workqueue_struct *bnxt_re_wq; -static void bnxt_re_ib_unreg (struct bnxt_re_dev *rdev); +static void bnxt_re_remove_device (struct bnxt_re_dev *rdev); +static void bnxt_re_ib_uninit (struct bnxt_re_dev *rdev);static void bnxt_re_destroy_chip_ctx (struct bnxt_re_dev *rdev) {
@@ -237,7 +238,9 @@ static void bnxt_re_shutdown(void *p)if (!rdev)return ; -bnxt_re_ib_unreg (rdev); +bnxt_re_ib_uninit (rdev); +ASSERT_RTNL (); +bnxt_re_remove_device (rdev); }static void bnxt_re_stop_irq (void *handle)
@@ -1317,7 +1320,37 @@ static void bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev)le16_to_cpu (resp.hwrm_intf_patch); } -static void bnxt_re_ib_unreg (struct bnxt_re_dev *rdev) +static void bnxt_re_ib_uninit (struct bnxt_re_dev *rdev) +{ +/* Cleanup ib dev */ +if (test_bit (BNXT_RE_FLAG_IBDEV_REGISTERED , &rdev->flags)) { +ib_unregister_device (&rdev->ibdev); +clear_bit (BNXT_RE_FLAG_IBDEV_REGISTERED , &rdev->flags); + } +} + +int bnxt_re_ib_init (struct bnxt_re_dev *rdev) +{ +int rc =0 ; + +/* Register ib dev */ + rc =bnxt_re_register_ib (rdev); +if (rc) { +pr_err ("Failed to register with IB: %#x\n" , rc); +return rc; + } +set_bit (BNXT_RE_FLAG_IBDEV_REGISTERED , &rdev->flags); +dev_info (rdev_to_dev (rdev),"Device registered successfully" ); +ib_get_eth_speed (&rdev->ibdev,1 , &rdev->active_speed, + &rdev->active_width); +set_bit (BNXT_RE_FLAG_ISSUE_ROCE_STATS , &rdev->flags); +bnxt_re_dispatch_event (&rdev->ibdev,NULL ,1 ,IB_EVENT_PORT_ACTIVE ); +bnxt_re_dispatch_event (&rdev->ibdev,NULL ,1 ,IB_EVENT_GID_CHANGE ); + +return rc; +} + +static void bnxt_re_dev_uninit (struct bnxt_re_dev *rdev) {u8 type;int rc;
@@ -1373,20 +1406,15 @@ static void bnxt_re_worker(struct work_struct *work)schedule_delayed_work (&rdev->worker,msecs_to_jiffies (30000 )); } -static int bnxt_re_ib_reg (struct bnxt_re_dev *rdev) +static int bnxt_re_dev_init (struct bnxt_re_dev *rdev) {struct bnxt_qplib_creq_ctx *creq;struct bnxt_re_ring_attr rattr;u32 db_offt; -bool locked;int vid;u8 type;int rc; -/* Acquire rtnl lock through out this function */ -rtnl_lock (); - locked = true; -/* Registered a new RoCE device instance to netdev */ memset (&rattr,0 ,sizeof (rattr)); rc =bnxt_re_register_netdev (rdev);
@@ -1514,23 +1542,6 @@ static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)schedule_delayed_work (&rdev->worker,msecs_to_jiffies (30000 )); } -rtnl_unlock (); - locked = false; - -/* Register ib dev */ - rc =bnxt_re_register_ib (rdev); -if (rc) { -ibdev_err (&rdev->ibdev, -"Failed to register with IB: %#x\n" , rc); - goto fail; - } -set_bit (BNXT_RE_FLAG_IBDEV_REGISTERED , &rdev->flags); -ibdev_info (&rdev->ibdev,"Device registered successfully" ); -ib_get_eth_speed (&rdev->ibdev,1 , &rdev->active_speed, - &rdev->active_width); -set_bit (BNXT_RE_FLAG_ISSUE_ROCE_STATS , &rdev->flags); -bnxt_re_dispatch_event (&rdev->ibdev,NULL ,1 ,IB_EVENT_PORT_ACTIVE ); -return 0 ; free_sctx:bnxt_re_net_stats_ctx_free (rdev, rdev->qplib_ctx.stats.fw_id);
@@ -1544,10 +1555,7 @@ static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev) free_rcfw:bnxt_qplib_free_rcfw_channel (&rdev->rcfw); fail: -if (!locked) -rtnl_lock (); -bnxt_re_ib_unreg (rdev); -rtnl_unlock (); +bnxt_re_dev_uninit (rdev);return rc; }
@@ -1589,9 +1597,35 @@ static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct net_device *netdev)return rc; } -static void bnxt_re_remove_one (struct bnxt_re_dev *rdev) +static void bnxt_re_remove_device (struct bnxt_re_dev *rdev) { +bnxt_re_dev_uninit (rdev);pci_dev_put (rdev->en_dev->pdev); +bnxt_re_dev_unreg (rdev); +} + +static int bnxt_re_add_device (struct bnxt_re_dev **rdev, +struct net_device *netdev) +{ +int rc; + + rc =bnxt_re_dev_reg (rdev, netdev); +if (rc == -ENODEV ) +return rc; +if (rc) { +pr_err ("Failed to register with the device %s: %#x\n" , + netdev->name, rc); +return rc; + } + +pci_dev_get ((*rdev)->en_dev->pdev); + rc =bnxt_re_dev_init (*rdev); +if (rc) { +pci_dev_put ((*rdev)->en_dev->pdev); +bnxt_re_dev_unreg (*rdev); + } + +return rc; }/* Handle all deferred netevents tasks */
@@ -1606,16 +1640,17 @@ static void bnxt_re_task(struct work_struct *work)if (re_work->event !=NETDEV_REGISTER && !test_bit (BNXT_RE_FLAG_IBDEV_REGISTERED , &rdev->flags)) -return ; + goto done;switch (re_work->event) {case NETDEV_REGISTER : - rc =bnxt_re_ib_reg (rdev); + rc =bnxt_re_ib_init (rdev);if (rc) {ibdev_err (&rdev->ibdev,"Failed to register with IB: %#x" , rc); -bnxt_re_remove_one (rdev); -bnxt_re_dev_unreg (rdev); +rtnl_lock (); +bnxt_re_remove_device (rdev); +rtnl_unlock (); goto exit; }break ;
@@ -1638,17 +1673,13 @@ static void bnxt_re_task(struct work_struct *work)default :break ; } +done:smp_mb__before_atomic ();atomic_dec (&rdev->sched_count); exit:kfree (re_work); } -static void bnxt_re_init_one (struct bnxt_re_dev *rdev) -{ -pci_dev_get (rdev->en_dev->pdev); -} -/* * "Notifier chain callback can be invoked for the same chain from * different CPUs at the same time".
@@ -1686,17 +1717,9 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier,case NETDEV_REGISTER :if (rdev)break ; - rc =bnxt_re_dev_reg (&rdev, real_dev); -if (rc == -ENODEV ) -break ; -if (rc) { -ibdev_err (&rdev->ibdev, -"Failed to register with the device %s: %#x\n" , - real_dev->name, rc); -break ; - } -bnxt_re_init_one (rdev); - sch_work = true; + rc =bnxt_re_add_device (&rdev, real_dev); +if (!rc) + sch_work = true;break ;case NETDEV_UNREGISTER :
@@ -1705,9 +1728,8 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier,*/ if (atomic_read (&rdev->sched_count) >0 ) goto exit; -bnxt_re_ib_unreg (rdev); -bnxt_re_remove_one (rdev); -bnxt_re_dev_unreg (rdev); +bnxt_re_ib_uninit (rdev); +bnxt_re_remove_device (rdev);break ;default :
@@ -1784,12 +1806,11 @@ static void __exit bnxt_re_mod_exit(void)*/ flush_workqueue (bnxt_re_wq);bnxt_re_dev_stop (rdev); +bnxt_re_ib_uninit (rdev);/* Acquire the rtnl_lock as the L2 resources are freed here */ rtnl_lock (); -bnxt_re_ib_unreg (rdev); +bnxt_re_remove_device (rdev);rtnl_unlock (); -bnxt_re_remove_one (rdev); -bnxt_re_dev_unreg (rdev); }unregister_netdevice_notifier (&bnxt_re_netdev_notifier);if (bnxt_re_wq)
Signature · 1 line
2.5.5
RE: [PATCH for-next v3 1/2] RDMA/bnxt_re: Refactor device add/remove functionalities
Hi Selvin,
Quoted 3 lines
From: [email protected] <linux-rdma- [email protected]> On Behalf Of Selvin Xavier
[..]
Quoted 15 lines
drivers/infiniband/hw/bnxt_re/main.c +15
-0+int bnxt_re_ib_init (struct bnxt_re_dev *rdev) { +int rc =0 ; + +/* Register ib dev */ + rc =bnxt_re_register_ib (rdev); +if (rc) { +pr_err ("Failed to register with IB: %#x\n" , rc); +return rc; + } +set_bit (BNXT_RE_FLAG_IBDEV_REGISTERED , &rdev->flags); +dev_info (rdev_to_dev (rdev),"Device registered successfully" ); +ib_get_eth_speed (&rdev->ibdev,1 , &rdev->active_speed, + &rdev->active_width); +set_bit (BNXT_RE_FLAG_ISSUE_ROCE_STATS , &rdev->flags); + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
Quoted 1 line
IB_EVENT_PORT_ACTIVE);
What if the link is down at this point?
I see that it was done this way before, but since you are refactoring, may be you want to relook? Do you still want to report it as active?
Quoted 3 lines
(quoted diff) +1
-0+ bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
IB_EVENT_GID_CHANGE);
+
GID addition, deletion decisions for RoCE ports are taken care by the IB core. So hw driver shouldn't report this event. Please remove this call.
Selvin Xavier
maintainerRe: [PATCH for-next v3 1/2] RDMA/bnxt_re: Refactor device add/remove functionalities
Quoted 18 lines · Parav Pandit
Hi Selvin,
Quoted 3 lines
From: [email protected] <linux-rdma- [email protected]> On Behalf Of Selvin Xavier
[..]
Quoted 11 lines
drivers/infiniband/hw/bnxt_re/main.c +11
-0+int bnxt_re_ib_init (struct bnxt_re_dev *rdev) { +int rc =0 ; + +/* Register ib dev */ + rc =bnxt_re_register_ib (rdev); +if (rc) { +pr_err ("Failed to register with IB: %#x\n" , rc); +return rc; + } +set_bit (BNXT_RE_FLAG_IBDEV_REGISTERED , &rdev->flags); +dev_info (rdev_to_dev (rdev),"Device registered successfully" );
Quoted 8 lines
Quoted 5 lines
drivers/infiniband/hw/bnxt_re/main.c +4
-0+ib_get_eth_speed (&rdev->ibdev,1 , &rdev->active_speed, + &rdev->active_width); +set_bit (BNXT_RE_FLAG_ISSUE_ROCE_STATS , &rdev->flags); + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
IB_EVENT_PORT_ACTIVE);What if the link is down at this point?
I see that it was done this way before, but since you are refactoring, may be you want to relook? Do you still want to report it as active?
No.. Will change it based on the link status. thanks
Quoted 6 lines
Quoted 4 lines
+ bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
IB_EVENT_GID_CHANGE);
+GID addition, deletion decisions for RoCE ports are taken care by the IB core. So hw driver shouldn't report this event. Please remove this call.
Will do.
Selvin Xavier
maintainerPATCH v3 2/2 RDMA/bnxt_re: Use driver_unregister and unregistration API
In reply to Selvin XavierUsing the new unregister APIs provided by the core. Provide the dealloc_driver hook for the core to callback at the time of device un-registration.
bnxt_re VF resources are created by the corresponding PF driver. During ib_unregister_driver, PF might get removed before VF and this could cause failure when VFs are removed. Driver is explicitly queuing the removal of VF devices before calling ib_unregister_driver.
--- drivers/infiniband/hw/bnxt_re/main.c | 106 ++++++++++++++--------------------- 1 file changed, 42 insertions(+), 64 deletions(-)
drivers/infiniband/hw/bnxt_re/main.c +42 -64
@@ -79,7 +79,8 @@ static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list);static DEFINE_MUTEX (bnxt_re_dev_lock);static struct workqueue_struct *bnxt_re_wq;static void bnxt_re_remove_device (struct bnxt_re_dev *rdev); -static void bnxt_re_ib_uninit (struct bnxt_re_dev *rdev); +static void bnxt_re_dealloc_driver (struct ib_device *ib_dev); +static void bnxt_re_stop_irq (void *handle);static void bnxt_re_destroy_chip_ctx (struct bnxt_re_dev *rdev) {
@@ -237,10 +238,10 @@ static void bnxt_re_shutdown(void *p)if (!rdev)return ; - -bnxt_re_ib_uninit (rdev);ASSERT_RTNL (); -bnxt_re_remove_device (rdev); +/* Release the MSIx vectors before queuing unregister */ +bnxt_re_stop_irq (rdev); +ib_unregister_device_queued (&rdev->ibdev); }static void bnxt_re_stop_irq (void *handle)
@@ -542,17 +543,12 @@ static bool is_bnxt_re_dev(struct net_device *netdev)static struct bnxt_re_dev *bnxt_re_from_netdev (struct net_device *netdev) { -struct bnxt_re_dev *rdev; +struct ib_device *ibdev = +ib_device_get_by_netdev (netdev,RDMA_DRIVER_BNXT_RE ); +if (!ibdev) +return NULL ; -rcu_read_lock (); -list_for_each_entry_rcu (rdev, &bnxt_re_dev_list, list) { -if (rdev->netdev == netdev) { -rcu_read_unlock (); -return rdev; - } - } -rcu_read_unlock (); -return NULL ; + return container_of(ibdev, struct bnxt_re_dev, ibdev); } static void bnxt_re_dev_unprobe(struct net_device *netdev,
@@ -626,11 +622,6 @@ static const struct attribute_group bnxt_re_dev_attr_group = { .attrs = bnxt_re_attributes, }; -static void bnxt_re_unregister_ib (struct bnxt_re_dev *rdev) -{ -ib_unregister_device (&rdev->ibdev); -} -static const struct ib_device_ops bnxt_re_dev_ops = { .owner =THIS_MODULE , .driver_id =RDMA_DRIVER_BNXT_RE ,
@@ -645,6 +636,7 @@ static const struct ib_device_ops bnxt_re_dev_ops = { .create_cq = bnxt_re_create_cq, .create_qp = bnxt_re_create_qp, .create_srq = bnxt_re_create_srq, + .dealloc_driver = bnxt_re_dealloc_driver, .dealloc_pd = bnxt_re_dealloc_pd, .dealloc_ucontext = bnxt_re_dealloc_ucontext, .del_gid = bnxt_re_del_gid,
@@ -741,15 +733,11 @@ static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev) {dev_put (rdev->netdev); rdev->netdev =NULL ; -mutex_lock (&bnxt_re_dev_lock);list_del_rcu (&rdev->list);mutex_unlock (&bnxt_re_dev_lock);synchronize_rcu (); - -ib_dealloc_device (&rdev->ibdev); -/* rdev is gone */ } static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
@@ -1320,15 +1308,6 @@ static void bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev)le16_to_cpu (resp.hwrm_intf_patch); } -static void bnxt_re_ib_uninit (struct bnxt_re_dev *rdev) -{ -/* Cleanup ib dev */ -if (test_bit (BNXT_RE_FLAG_IBDEV_REGISTERED , &rdev->flags)) { -ib_unregister_device (&rdev->ibdev); -clear_bit (BNXT_RE_FLAG_IBDEV_REGISTERED , &rdev->flags); - } -} -int bnxt_re_ib_init (struct bnxt_re_dev *rdev) {int rc =0 ;
@@ -1355,10 +1334,6 @@ static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev)u8 type;int rc; -if (test_and_clear_bit (BNXT_RE_FLAG_IBDEV_REGISTERED , &rdev->flags)) { -/* Cleanup ib dev */ -bnxt_re_unregister_ib (rdev); - }if (test_and_clear_bit (BNXT_RE_FLAG_QOS_WORK_REG , &rdev->flags))cancel_delayed_work_sync (&rdev->worker);
@@ -1628,6 +1603,19 @@ static int bnxt_re_add_device(struct bnxt_re_dev **rdev,return rc; } +static void bnxt_re_dealloc_driver (struct ib_device *ib_dev) +{ +struct bnxt_re_dev *rdev = + container_of(ib_dev, struct bnxt_re_dev, ibdev); + +clear_bit (BNXT_RE_FLAG_IBDEV_REGISTERED , &rdev->flags); +dev_info (rdev_to_dev (rdev),"Unregistering Device" ); + +rtnl_lock (); +bnxt_re_remove_device (rdev); +rtnl_unlock (); +} +/* Handle all deferred netevents tasks */ static void bnxt_re_task (struct work_struct *work) {
@@ -1702,6 +1690,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier,struct bnxt_re_dev *rdev;int rc =0 ;bool sch_work = false; +bool release = true; real_dev =rdma_vlan_dev_real_dev (netdev);if (!real_dev)
@@ -1709,7 +1698,8 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, rdev =bnxt_re_from_netdev (real_dev);if (!rdev && event !=NETDEV_REGISTER ) - goto exit; +return NOTIFY_OK ; +if (real_dev != netdev) goto exit;
@@ -1720,6 +1710,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, rc =bnxt_re_add_device (&rdev, real_dev);if (!rc) sch_work = true; + release = false;break ;case NETDEV_UNREGISTER :
@@ -1728,8 +1719,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier,*/ if (atomic_read (&rdev->sched_count) >0 ) goto exit; -bnxt_re_ib_uninit (rdev); -bnxt_re_remove_device (rdev); +ib_unregister_device_queued (&rdev->ibdev);break ;default :
@@ -1751,6 +1741,8 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, } exit: +if (rdev && release) +ib_device_put (&rdev->ibdev);return NOTIFY_DONE ; }
@@ -1786,35 +1778,21 @@ static int __init bnxt_re_mod_init(void) static void __exit bnxt_re_mod_exit(void) { -struct bnxt_re_dev *rdev, *next; -LIST_HEAD (to_be_deleted); +struct bnxt_re_dev *rdev; -mutex_lock (&bnxt_re_dev_lock); -/* Free all adapter allocated resources */ -if (!list_empty (&bnxt_re_dev_list)) -list_splice_init (&bnxt_re_dev_list, &to_be_deleted); -mutex_unlock (&bnxt_re_dev_lock); -/* -* Cleanup the devices in reverse order so that the VF device -* cleanup is done before PF cleanup -*/ -list_for_each_entry_safe_reverse (rdev, next, &to_be_deleted, list) { -ibdev_info (&rdev->ibdev,"Unregistering Device" ); -/* -* Flush out any scheduled tasks before destroying the -* resources -*/ -flush_workqueue (bnxt_re_wq); -bnxt_re_dev_stop (rdev); -bnxt_re_ib_uninit (rdev); -/* Acquire the rtnl_lock as the L2 resources are freed here */ -rtnl_lock (); -bnxt_re_remove_device (rdev); -rtnl_unlock (); - }unregister_netdevice_notifier (&bnxt_re_netdev_notifier);if (bnxt_re_wq)destroy_workqueue (bnxt_re_wq); +list_for_each_entry (rdev, &bnxt_re_dev_list, list) { +/* VF device removal should be called before the removal +* of PF device. Queue VFs unregister first, so that VFs +* shall be removed before the PF during the call of +* ib_unregister_driver. +*/ +if (rdev->is_virtfn) +ib_unregister_device (&rdev->ibdev); + } +ib_unregister_driver (RDMA_DRIVER_BNXT_RE ); }module_init (bnxt_re_mod_init);
Signature · 1 line
2.5.5