RDMA/mana_ib: Enable RNIC adapter and populate it with GIDs
Subsystems (2)
v3 cover RDMA/mana_ib: Enable RNIC adapter and populate it with GIDs
From: Konstantin Taranov <[email protected]>
This patch series creates RNIC adapter in mana_ib.
To create the adapter, we must create one EQ.
In the future patches, this EQ will be used for fatal RC QP error events. In the future patches, we will also add more EQs for CQs.
mana_ib is served by mana ethernet for RAW QPs and by RNIC for RC QPs. If RNIC adapter cannot be created in the HW, we fail loading mana_ib. RNIC is available only for port 1.
As a minimal usage, this patch series brings adding and removing GIDs. For this, we set master netdev to the ib device and set required port parameters to get GIDs. RNIC of mana supports IPv6 and IPv4 addresses that are stored in the HW. The MAC address is also stored in the RNIC.
v2->v3:
* On RNIC creation errors, we fail the loading. Removed corresponding checks.
* Revised commit messages.
* Removed unused members of new enums.
* Added a patch to set mac address to the series
v1->v2:
* Fixed rcu_read_unlock() and updated commit message in "Enable RoCE on port 1"
Konstantin Taranov (6):
RDMA/mana_ib: Add EQ creation for rnic adapter
RDMA/mana_ib: Create and destroy rnic adapter
RDMA/mana_ib: implement port parameters
RDMA/mana_ib: enable RoCE on port 1
RDMA/mana_ib: adding and deleting GIDs
RDMA/mana_ib: Configure mac address in RNIC
drivers/infiniband/hw/mana/device.c | 48 ++++++- drivers/infiniband/hw/mana/main.c | 203 ++++++++++++++++++++++++++- drivers/infiniband/hw/mana/mana_ib.h | 87 ++++++++++++ 3 files changed, 330 insertions(+), 8 deletions(-)
base-commit: f10242b3da908dc9d4bfa040e6511a5b86522499
Signature · 1 line
2.43.0
PATCH v3 1/6 RDMA/mana_ib: Add EQ creation for rnic adapter
From: Konstantin Taranov <[email protected]>
Create an error EQ for the RNIC adapter.
--- drivers/infiniband/hw/mana/device.c | 13 ++++++++++--- drivers/infiniband/hw/mana/main.c | 26 ++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 5 +++++ 3 files changed, 41 insertions(+), 3 deletions(-)
drivers/infiniband/hw/mana/device.c +10 -3
@@ -92,15 +92,23 @@ static int mana_ib_probe(struct auxiliary_device *adev, goto deregister_device; } + ret =mana_ib_create_eqs (dev); +if (ret) { +ibdev_err (&dev->ib_dev,"Failed to create EQs, ret %d" , ret); + goto deregister_device; + } + ret =ib_register_device (&dev->ib_dev,"mana_%d" , mdev->gdma_context->dev);if (ret) - goto deregister_device; + goto destroy_eqs;dev_set_drvdata (&adev->dev, dev);return 0 ; +destroy_eqs: +mana_ib_destroy_eqs (dev); deregister_device:mana_gd_deregister_device (dev->gdma_dev); free_ib_device:
@@ -113,9 +121,8 @@ static void mana_ib_remove(struct auxiliary_device *adev)struct mana_ib_dev *dev =dev_get_drvdata (&adev->dev);ib_unregister_device (&dev->ib_dev); - +mana_ib_destroy_eqs (dev);mana_gd_deregister_device (dev->gdma_dev); -ib_dealloc_device (&dev->ib_dev); }
drivers/infiniband/hw/mana/main.c +26 -0
@@ -613,3 +613,29 @@ int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)return 0 ; } + +int mana_ib_create_eqs (struct mana_ib_dev *mdev) +{ +struct gdma_context *gc =mdev_to_gc (mdev); +struct gdma_queue_spec spec = {}; +int err; + + spec.type =GDMA_EQ ; + spec.monitor_avl_buf = false; + spec.queue_size =EQ_SIZE ; + spec.eq.callback =NULL ; + spec.eq.context = mdev; + spec.eq.log2_throttle_limit =LOG2_EQ_THROTTLE ; + spec.eq.msix_index =0 ; + + err =mana_gd_create_mana_eq (&gc->mana_ib, &spec, &mdev->fatal_err_eq); +if (err) +return err; + +return 0 ; +} + +void mana_ib_destroy_eqs (struct mana_ib_dev *mdev) +{ +mana_gd_destroy_queue (mdev_to_gc (mdev), mdev->fatal_err_eq); +}
drivers/infiniband/hw/mana/mana_ib.h +5 -0
@@ -54,6 +54,7 @@ struct mana_ib_queue {struct mana_ib_dev {struct ib_device ib_dev;struct gdma_dev *gdma_dev; +struct gdma_queue *fatal_err_eq;struct mana_ib_adapter_caps adapter_caps; };
@@ -233,4 +234,8 @@ int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,void mana_ib_disassociate_ucontext (struct ib_ucontext *ibcontext);int mana_ib_gd_query_adapter_caps (struct mana_ib_dev *mdev); + +int mana_ib_create_eqs (struct mana_ib_dev *mdev); + +void mana_ib_destroy_eqs (struct mana_ib_dev *mdev);#endif
Signature · 1 line
2.43.0
PATCH v3 2/6 RDMA/mana_ib: Create and destroy rnic adapter
In reply to Konstantin TaranovFrom: Konstantin Taranov <[email protected]>
Add functions for RNIC creation and destruction.
If creation fails, the ib_probe fails as well.
--- drivers/infiniband/hw/mana/device.c | 9 +++++- drivers/infiniband/hw/mana/main.c | 43 ++++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 28 ++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-)
drivers/infiniband/hw/mana/device.c +8 -1
@@ -98,15 +98,21 @@ static int mana_ib_probe(struct auxiliary_device *adev, goto deregister_device; } + ret =mana_ib_gd_create_rnic_adapter (dev); +if (ret) + goto destroy_eqs; + ret =ib_register_device (&dev->ib_dev,"mana_%d" , mdev->gdma_context->dev);if (ret) - goto destroy_eqs; + goto destroy_rnic;dev_set_drvdata (&adev->dev, dev);return 0 ; +destroy_rnic: +mana_ib_gd_destroy_rnic_adapter (dev); destroy_eqs:mana_ib_destroy_eqs (dev); deregister_device:
@@ -121,6 +127,7 @@ static void mana_ib_remove(struct auxiliary_device *adev)struct mana_ib_dev *dev =dev_get_drvdata (&adev->dev);ib_unregister_device (&dev->ib_dev); +mana_ib_gd_destroy_rnic_adapter (dev);mana_ib_destroy_eqs (dev);mana_gd_deregister_device (dev->gdma_dev);ib_dealloc_device (&dev->ib_dev);
drivers/infiniband/hw/mana/main.c +43 -0
46 lines not on this page · raw message
drivers/infiniband/hw/mana/mana_ib.h +28 -0
@@ -54,6 +54,7 @@ struct mana_ib_queue {struct mana_ib_dev {struct ib_device ib_dev;struct gdma_dev *gdma_dev; +mana_handle_t adapter_handle;struct gdma_queue *fatal_err_eq;struct mana_ib_adapter_caps adapter_caps; };
@@ -113,6 +114,8 @@ struct mana_ib_rwq_ind_table {enum mana_ib_command_code {MANA_IB_GET_ADAPTER_CAP =0x30001 , +MANA_IB_CREATE_ADAPTER =0x30002 , +MANA_IB_DESTROY_ADAPTER =0x30003 , };struct mana_ib_query_adapter_caps_req {
@@ -141,6 +144,27 @@ struct mana_ib_query_adapter_caps_resp {u32 max_inline_data_size; };/* HW Data */ +struct mana_rnic_create_adapter_req { +struct gdma_req_hdr hdr; +u32 notify_eq_id; +u32 reserved; +u64 feature_flags; +};/*HW Data */ + +struct mana_rnic_create_adapter_resp { +struct gdma_resp_hdr hdr; +mana_handle_t adapter; +};/* HW Data */ + +struct mana_rnic_destroy_adapter_req { +struct gdma_req_hdr hdr; +mana_handle_t adapter; +};/*HW Data */ + +struct mana_rnic_destroy_adapter_resp { +struct gdma_resp_hdr hdr; +};/* HW Data */ +static inline struct gdma_context *mdev_to_gc (struct mana_ib_dev *mdev) {return mdev->gdma_dev->gdma_context;
@@ -238,4 +262,8 @@ int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *mdev);int mana_ib_create_eqs (struct mana_ib_dev *mdev);void mana_ib_destroy_eqs (struct mana_ib_dev *mdev); + +int mana_ib_gd_create_rnic_adapter (struct mana_ib_dev *mdev); + +int mana_ib_gd_destroy_rnic_adapter (struct mana_ib_dev *mdev);#endif
Signature · 1 line
2.43.0
PATCH v3 3/6 RDMA/mana_ib: implement port parameters
In reply to Konstantin TaranovFrom: Konstantin Taranov <[email protected]>
Implement port parameters for RNIC:
1) extend query_port() method
2) implement get_link_layer()
3) implement query_pkey()
Only port 1 can store GIDs.
--- drivers/infiniband/hw/mana/device.c | 2 ++ drivers/infiniband/hw/mana/main.c | 37 +++++++++++++++++++++++++++- drivers/infiniband/hw/mana/mana_ib.h | 4 +++ 3 files changed, 42 insertions(+), 1 deletion(-)
drivers/infiniband/hw/mana/device.c +2 -0
@@ -29,12 +29,14 @@ static const struct ib_device_ops mana_ib_dev_ops = { .destroy_rwq_ind_table = mana_ib_destroy_rwq_ind_table, .destroy_wq = mana_ib_destroy_wq, .disassociate_ucontext = mana_ib_disassociate_ucontext, + .get_link_layer = mana_ib_get_link_layer, .get_port_immutable = mana_ib_get_port_immutable, .mmap = mana_ib_mmap, .modify_qp = mana_ib_modify_qp, .modify_wq = mana_ib_modify_wq, .query_device = mana_ib_query_device, .query_gid = mana_ib_query_gid, + .query_pkey = mana_ib_query_pkey, .query_port = mana_ib_query_port, .reg_user_mr = mana_ib_reg_user_mr,
drivers/infiniband/hw/mana/main.c +36 -1
@@ -557,7 +557,42 @@ int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props,int mana_ib_query_port (struct ib_device *ibdev,u32 port,struct ib_port_attr *props) { -/* This version doesn't return port properties */ +struct net_device *ndev =mana_ib_get_netdev (ibdev, port); + +if (!ndev) +return -EINVAL ; + +memset (props,0 ,sizeof (*props)); + props->max_mtu =IB_MTU_4096 ; + props->active_mtu =ib_mtu_int_to_enum (ndev->mtu); + +if (netif_carrier_ok (ndev) &&netif_running (ndev)) { + props->state =IB_PORT_ACTIVE ; + props->phys_state =IB_PORT_PHYS_STATE_LINK_UP ; + }else { + props->state =IB_PORT_DOWN ; + props->phys_state =IB_PORT_PHYS_STATE_DISABLED ; + } + + props->active_width =IB_WIDTH_4X ; + props->active_speed =IB_SPEED_EDR ; + props->pkey_tbl_len =1 ; +if (port ==1 ) + props->gid_tbl_len =16 ; + +return 0 ; +} + +enum rdma_link_layer mana_ib_get_link_layer (struct ib_device *device,u32 port_num) +{ +return IB_LINK_LAYER_ETHERNET ; +} + +int mana_ib_query_pkey (struct ib_device *ibdev,u32 port,u16 index,u16 *pkey) +{ +if (index !=0 ) +return -EINVAL ; + *pkey =IB_DEFAULT_PKEY_FULL ;return 0 ; }
drivers/infiniband/hw/mana/mana_ib.h +4 -0
@@ -266,4 +266,8 @@ void mana_ib_destroy_eqs(struct mana_ib_dev *mdev);int mana_ib_gd_create_rnic_adapter (struct mana_ib_dev *mdev);int mana_ib_gd_destroy_rnic_adapter (struct mana_ib_dev *mdev); + +int mana_ib_query_pkey (struct ib_device *ibdev,u32 port,u16 index,u16 *pkey); + +enum rdma_link_layer mana_ib_get_link_layer (struct ib_device *device,u32 port_num);#endif
Signature · 1 line
2.43.0
PATCH v3 4/6 RDMA/mana_ib: enable RoCE on port 1
In reply to Konstantin TaranovFrom: Konstantin Taranov <[email protected]>
Set netdev and RoCEv2 flag to enable GID population on port 1. Use GIDs of the master netdev. As mc->ports[] stores slave devices, use a helper to get the master netdev.
--- drivers/infiniband/hw/mana/device.c | 15 +++++++++++++++ drivers/infiniband/hw/mana/main.c | 15 +++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-)
drivers/infiniband/hw/mana/device.c +15 -0
@@ -53,6 +53,7 @@ static int mana_ib_probe(struct auxiliary_device *adev, { struct mana_adev *madev = container_of(adev, struct mana_adev, adev);struct gdma_dev *mdev = madev->mdev; +struct net_device *upper_ndev;struct mana_context *mc;struct mana_ib_dev *dev;int ret;
@@ -79,6 +80,20 @@ static int mana_ib_probe(struct auxiliary_device *adev, dev->ib_dev.num_comp_vectors =1 ; dev->ib_dev.dev.parent = mdev->gdma_context->dev; +rcu_read_lock ();/* required to get upper dev */ + upper_ndev =netdev_master_upper_dev_get_rcu (mc->ports[0 ]); +if (!upper_ndev) { +rcu_read_unlock (); +ibdev_err (&dev->ib_dev,"Failed to get master netdev" ); + goto free_ib_device; + } + ret =ib_device_set_netdev (&dev->ib_dev, upper_ndev,1 ); +rcu_read_unlock (); +if (ret) { +ibdev_err (&dev->ib_dev,"Failed to set ib netdev, ret %d" , ret); + goto free_ib_device; + } + ret =mana_gd_register_device (&mdev->gdma_context->mana_ib);if (ret) { ibdev_err(&dev->ib_dev, "Failed to register device, ret %d",
drivers/infiniband/hw/mana/main.c +11 -4
@@ -527,11 +527,18 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)int mana_ib_get_port_immutable (struct ib_device *ibdev,u32 port_num,struct ib_port_immutable *immutable) { -/* -* This version only support RAW_PACKET -* other values need to be filled for other types -*/ +struct ib_port_attr attr; +int err; + + err =ib_query_port (ibdev, port_num, &attr); +if (err) +return err; + + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; immutable->core_cap_flags =RDMA_CORE_PORT_RAW_PACKET ; +if (port_num ==1 ) + immutable->core_cap_flags |=RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP ;return 0 ; }
Signature · 1 line
2.43.0
Re: [PATCH rdma-next v3 4/6] RDMA/mana_ib: enable RoCE on port 1
Hi Konstantin,
Quoted 25 lines · Konstantin Taranov
From: Konstantin Taranov <[email protected]>
Set netdev and RoCEv2 flag to enable GID population on port 1. Use GIDs of the master netdev. As mc->ports[] stores slave devices, use a helper to get the master netdev.
Signed-off-by Konstantin Taranov <[email protected]>--- drivers/infiniband/hw/mana/device.c | 15 +++++++++++++++ drivers/infiniband/hw/mana/main.c | 15 +++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-)drivers/infiniband/hw/mana/device.c +1
-0@@ -53,6 +53,7 @@ static int mana_ib_probe(struct auxiliary_device *adev, { struct mana_adev *madev = container_of(adev, struct mana_adev, adev);struct gdma_dev *mdev = madev->mdev; +struct net_device *upper_ndev;struct mana_context *mc;struct mana_ib_dev *dev;int ret;
drivers/infiniband/hw/mana/device.c static int mana_ib_probe(struct auxiliary_device *adev, · line 89
Quoted 10 lines
drivers/infiniband/hw/mana/device.c +7
-0@@ -79,6 +80,20 @@ static int mana_ib_probe(struct auxiliary_device *adev, dev->ib_dev.num_comp_vectors =1 ; dev->ib_dev.dev.parent = mdev->gdma_context->dev; +rcu_read_lock ();/* required to get upper dev */ + upper_ndev =netdev_master_upper_dev_get_rcu (mc->ports[0 ]); +if (!upper_ndev) { +rcu_read_unlock (); +ibdev_err (&dev->ib_dev,"Failed to get master netdev" ); + goto free_ib_device; + }
Clang now warns (or errors with CONFIG_WERROR):
drivers/infiniband/hw/mana/device.c:88:6: error: variable 'ret' is used uninitialized whenever 'if' condition is true [-Werror,-Wsometimes-uninitialized]
88 | if (!upper_ndev) {
| ^~~~~~~~~~~
drivers/infiniband/hw/mana/device.c:150:9: note: uninitialized use occurs here
150 | return ret;
| ^~~
drivers/infiniband/hw/mana/device.c:88:2: note: remove the 'if' if its condition is always false
88 | if (!upper_ndev) {
| ^~~~~~~~~~~~~~~~~~
89 | rcu_read_unlock();
| ~~~~~~~~~~~~~~~~~~
90 | ibdev_err(&dev->ib_dev, "Failed to get master netdev");
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
91 | goto free_ib_device;
| ~~~~~~~~~~~~~~~~~~~~
92 | }
| ~
drivers/infiniband/hw/mana/device.c:62:9: note: initialize the variable 'ret' to silence this warning
62 | int ret;
| ^
| = 01 error generated.
I could not really find a consistent return code for when netdev_master_upper_dev_get_rcu() fails. -ENODEV?
Cheers,
Nathan
Quoted 40 lines
drivers/infiniband/hw/mana/device.c +7
-0+ ret =ib_device_set_netdev (&dev->ib_dev, upper_ndev,1 ); +rcu_read_unlock (); +if (ret) { +ibdev_err (&dev->ib_dev,"Failed to set ib netdev, ret %d" , ret); + goto free_ib_device; + } + ret =mana_gd_register_device (&mdev->gdma_context->mana_ib);if (ret) { ibdev_err(&dev->ib_dev, "Failed to register device, ret %d",drivers/infiniband/hw/mana/main.c +11
-4@@ -527,11 +527,18 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)int mana_ib_get_port_immutable (struct ib_device *ibdev,u32 port_num,struct ib_port_immutable *immutable) { -/* -* This version only support RAW_PACKET -* other values need to be filled for other types -*/ +struct ib_port_attr attr; +int err; + + err =ib_query_port (ibdev, port_num, &attr); +if (err) +return err; + + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; immutable->core_cap_flags =RDMA_CORE_PORT_RAW_PACKET ; +if (port_num ==1 ) + immutable->core_cap_flags |=RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP ;return 0 ; }Signature · 1 line
2.43.0
Konstantin Taranov
maintainerRe: [PATCH rdma-next v3 4/6] RDMA/mana_ib: enable RoCE on port 1
Quoted 71 lines
Hi Konstantin,
Quoted 18 lines · Konstantin Taranov
From: Konstantin Taranov <[email protected]>
Set netdev and RoCEv2 flag to enable GID population on port 1. Use GIDs of the master netdev. As mc->ports[] stores slave devices, use a helper to get the master netdev.
Signed-off-by Konstantin Taranov <[email protected]>--- drivers/infiniband/hw/mana/device.c | 15 +++++++++++++++ drivers/infiniband/hw/mana/main.c | 15 +++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-)drivers/infiniband/hw/mana/device.c +0
-0drivers/infiniband/hw/mana/device.c static int mana_ib_probe(struct auxiliary_device · line 54
Quoted 2 lines
drivers/infiniband/hw/mana/device.c +0
-0@@ -53,6 +53,7 @@ static int mana_ib_probe(struct auxiliary_device *adev, { struct mana_adev *madev = container_of(adev, struct mana_adev,adev);
drivers/infiniband/hw/mana/device.c static int mana_ib_probe(struct auxiliary_device
Quoted 6 lines
drivers/infiniband/hw/mana/device.c +1
-0struct gdma_dev *mdev = madev->mdev; +struct net_device *upper_ndev;struct mana_context *mc;struct mana_ib_dev *dev;int ret;@@ -79,6 +80,20 @@ static int mana_ib_probe(struct auxiliary_device*adev,
drivers/infiniband/hw/mana/device.c static int mana_ib_probe(struct auxiliary_device *adev, · line 89
Quoted 10 lines
drivers/infiniband/hw/mana/device.c +7
-0dev->ib_dev.num_comp_vectors =1 ; dev->ib_dev.dev.parent = mdev->gdma_context->dev; +rcu_read_lock ();/* required to get upper dev */ + upper_ndev =netdev_master_upper_dev_get_rcu (mc->ports[0 ]); +if (!upper_ndev) { +rcu_read_unlock (); +ibdev_err (&dev->ib_dev,"Failed to get master netdev" ); + goto free_ib_device; + }Clang now warns (or errors with
CONFIG_WERROR):
drivers/infiniband/hw/mana/device.c:88:6: error: variable 'ret' is used
uninitialized whenever 'if' condition is true [-Werror,-Wsometimes- uninitialized]88 | if (!upper_ndev) { | ^~~~~~~~~~~ drivers/infiniband/hw/mana/device.c:150:9: note: uninitialized use occurs here 150 | return ret; | ^~~ drivers/infiniband/hw/mana/device.c:88:2: note: remove the 'if' if its condition is always false 88 | if (!upper_ndev) { | ^~~~~~~~~~~~~~~~~~ 89 | rcu_read_unlock(); | ~~~~~~~~~~~~~~~~~~ 90 | ibdev_err(&dev->ib_dev, "Failed to get master netdev"); | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 91 | goto free_ib_device; | ~~~~~~~~~~~~~~~~~~~~ 92 | } | ~ drivers/infiniband/hw/mana/device.c:62:9: note: initialize the variable 'ret' to silence this warning 62 | int ret; | ^ | = 0
Quoted 4 lines
1 error generated.
I could not really find a consistent return code for when
netdev_master_upper_dev_get_rcu()fails.-ENODEV?
Thanks for catching this! Yes, I think ret = -ENODEV; is appropriate fix. Should I send a patch to rdma-next? Or what should I do now to fix this?
Konstantin
Quoted 4 lines
Cheers,
Nathan
Re: [PATCH rdma-next v3 4/6] RDMA/mana_ib: enable RoCE on port 1
Quoted 6 lines · Konstantin Taranov
Quoted 2 lines
I could not really find a consistent return code for when
netdev_master_upper_dev_get_rcu()fails.-ENODEV?Thanks for catching this! Yes, I think ret =
-ENODEV; is appropriate fix. Should I send a patch to rdma-next? Or what should I do now to fix this?
Send a patch
Jason
PATCH v3 5/6 RDMA/mana_ib: adding and deleting GIDs
In reply to Konstantin TaranovFrom: Konstantin Taranov <[email protected]>
Implement add_gid and del_gid for RNIC.
IPv4 and IPv6 addresses are supported.
--- drivers/infiniband/hw/mana/device.c | 2 + drivers/infiniband/hw/mana/main.c | 60 ++++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 35 ++++++++++++++++ 3 files changed, 97 insertions(+)
drivers/infiniband/hw/mana/device.c +2 -0
@@ -15,6 +15,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { .driver_id = RDMA_DRIVER_MANA, .uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION, + .add_gid = mana_ib_gd_add_gid, .alloc_pd = mana_ib_alloc_pd, .alloc_ucontext = mana_ib_alloc_ucontext, .create_cq = mana_ib_create_cq,
@@ -23,6 +24,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { .create_wq = mana_ib_create_wq, .dealloc_pd = mana_ib_dealloc_pd, .dealloc_ucontext = mana_ib_dealloc_ucontext, + .del_gid = mana_ib_gd_del_gid, .dereg_mr = mana_ib_dereg_mr, .destroy_cq = mana_ib_destroy_cq, .destroy_qp = mana_ib_destroy_qp,
drivers/infiniband/hw/mana/main.c +60 -0
63 lines not on this page · raw message
drivers/infiniband/hw/mana/mana_ib.h +35 -0
@@ -116,6 +116,7 @@ enum mana_ib_command_code {MANA_IB_GET_ADAPTER_CAP =0x30001 ,MANA_IB_CREATE_ADAPTER =0x30002 ,MANA_IB_DESTROY_ADAPTER =0x30003 , + MANA_IB_CONFIG_IP_ADDR = 0x30004, }; struct mana_ib_query_adapter_caps_req {
@@ -165,6 +166,28 @@ struct mana_rnic_destroy_adapter_resp {struct gdma_resp_hdr hdr; };/* HW Data */ +enum mana_ib_addr_op { +ADDR_OP_ADD =1 , +ADDR_OP_REMOVE =2 , +}; + +enum sgid_entry_type { +SGID_TYPE_IPV4 =1 , +SGID_TYPE_IPV6 =2 , +}; + +struct mana_rnic_config_addr_req { +struct gdma_req_hdr hdr; +mana_handle_t adapter; +enum mana_ib_addr_op op; +enum sgid_entry_type sgid_type; +u8 ip_addr[16 ]; +};/* HW Data */ + +struct mana_rnic_config_addr_resp { +struct gdma_resp_hdr hdr; +};/* HW Data */ +static inline struct gdma_context *mdev_to_gc (struct mana_ib_dev *mdev) {return mdev->gdma_dev->gdma_context;
@@ -181,6 +204,14 @@ static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32return mc->ports[port -1 ]; } +static inline void copy_in_reverse (u8 *dst,const u8 *src,u32 size) +{ +u32 i; + +for (i =0 ; i < size; i++) + dst[size -1 - i] = src[i]; +} +int mana_ib_install_cq_cb (struct mana_ib_dev *mdev,struct mana_ib_cq *cq); int mana_ib_create_zero_offset_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
@@ -270,4 +301,8 @@ int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev);int mana_ib_query_pkey (struct ib_device *ibdev,u32 port,u16 index,u16 *pkey);enum rdma_link_layer mana_ib_get_link_layer (struct ib_device *device,u32 port_num); + +int mana_ib_gd_add_gid (const struct ib_gid_attr *attr,void **context); + +int mana_ib_gd_del_gid (const struct ib_gid_attr *attr,void **context);#endif
Signature · 1 line
2.43.0
PATCH v3 6/6 RDMA/mana_ib: Configure mac address in RNIC
In reply to Konstantin TaranovFrom: Konstantin Taranov <[email protected]>
Set local mac address in RNIC, which is required by the HW.
--- drivers/infiniband/hw/mana/device.c | 9 +++++++++ drivers/infiniband/hw/mana/main.c | 22 ++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 15 +++++++++++++++ 3 files changed, 46 insertions(+)
drivers/infiniband/hw/mana/device.c +9 -0
@@ -58,6 +58,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,struct net_device *upper_ndev;struct mana_context *mc;struct mana_ib_dev *dev; +u8 mac_addr[ETH_ALEN ];int ret; mc = mdev->driver_data;
@@ -89,6 +90,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,ibdev_err (&dev->ib_dev,"Failed to get master netdev" ); goto free_ib_device; } +ether_addr_copy (mac_addr, upper_ndev->dev_addr); ret =ib_device_set_netdev (&dev->ib_dev, upper_ndev,1 );rcu_read_unlock ();if (ret) {
@@ -121,6 +123,13 @@ static int mana_ib_probe(struct auxiliary_device *adev,if (ret) goto destroy_eqs; + ret =mana_ib_gd_config_mac (dev,ADDR_OP_ADD , mac_addr); +if (ret) { +ibdev_err (&dev->ib_dev,"Failed to add Mac address, ret %d" , + ret); + goto destroy_rnic; + } + ret =ib_register_device (&dev->ib_dev,"mana_%d" , mdev->gdma_context->dev);if (ret)
drivers/infiniband/hw/mana/main.c +22 -0
@@ -784,3 +784,25 @@ int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context)return 0 ; } + +int mana_ib_gd_config_mac (struct mana_ib_dev *mdev,enum mana_ib_addr_op op,u8 *mac) +{ +struct mana_rnic_config_mac_addr_resp resp = {}; +struct mana_rnic_config_mac_addr_req req = {}; +struct gdma_context *gc =mdev_to_gc (mdev); +int err; + +mana_gd_init_req_hdr (&req.hdr,MANA_IB_CONFIG_MAC_ADDR ,sizeof (req),sizeof (resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + req.op = op; +copy_in_reverse (req.mac_addr, mac,ETH_ALEN ); + + err =mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); +if (err) { +ibdev_err (&mdev->ib_dev,"Failed to config Mac addr err %d" , err); +return err; + } + +return 0 ; +}
drivers/infiniband/hw/mana/mana_ib.h +15 -0
@@ -117,6 +117,7 @@ enum mana_ib_command_code {MANA_IB_CREATE_ADAPTER =0x30002 ,MANA_IB_DESTROY_ADAPTER =0x30003 ,MANA_IB_CONFIG_IP_ADDR =0x30004 , + MANA_IB_CONFIG_MAC_ADDR = 0x30005, }; struct mana_ib_query_adapter_caps_req {
@@ -188,6 +189,18 @@ struct mana_rnic_config_addr_resp {struct gdma_resp_hdr hdr; };/* HW Data */ +struct mana_rnic_config_mac_addr_req { +struct gdma_req_hdr hdr; +mana_handle_t adapter; +enum mana_ib_addr_op op; +u8 mac_addr[ETH_ALEN ]; +u8 reserved[6 ]; +};/* HW Data */ + +struct mana_rnic_config_mac_addr_resp { +struct gdma_resp_hdr hdr; +};/* HW Data */ +static inline struct gdma_context *mdev_to_gc (struct mana_ib_dev *mdev) {return mdev->gdma_dev->gdma_context;
@@ -305,4 +318,6 @@ enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_nint mana_ib_gd_add_gid (const struct ib_gid_attr *attr,void **context);int mana_ib_gd_del_gid (const struct ib_gid_attr *attr,void **context); + +int mana_ib_gd_config_mac (struct mana_ib_dev *mdev,enum mana_ib_addr_op op,u8 *mac);#endif
Signature · 1 line
2.43.0
Leon Romanovsky
AppliedQuoted 8 lines · Konstantin Taranov
From: Konstantin Taranov <[email protected]>
This patch series creates RNIC adapter in
mana_ib.
To create the adapter, we must create one EQ.
In the future patches, this EQ will be used for fatal RC QP error events. In the future patches, we will also add more EQs for CQs.[...]
Applied, thanks!
[1/6] RDMA/mana_ib: Add EQ creation for rnic adapter
https://git.kernel.org/rdma/rdma/c/98b889c43935c4 [2/6] RDMA/mana_ib: Create and destroy rnic adapter https://git.kernel.org/rdma/rdma/c/1a79c2b9d4a087 [3/6] RDMA/mana_ib: implement port parameters https://git.kernel.org/rdma/rdma/c/4bda1d5332ec1b [4/6] RDMA/mana_ib: enable RoCE on port 1 https://git.kernel.org/rdma/rdma/c/8b184e4f1c328d [5/6] RDMA/mana_ib: adding and deleting GIDs https://git.kernel.org/rdma/rdma/c/faafb8b126ad60 [6/6] RDMA/mana_ib: Configure mac address in RNIC https://git.kernel.org/rdma/rdma/c/8859f009ace237
Best regards,
Signature · 1 line
Leon Romanovsky <[email protected]>