RDMA/mana_ib: Enable RNIC adapter and populate it with GIDs
Subsystems (2)
v2 cover RDMA/mana_ib: Enable RNIC adapter and populate it with GIDs
This patch series creates RNIC adapter in mana_ib.
To create the adapter, we must create one EQ.
In the future patches, this EQ will be used for fatal RC QP error events. In the future patches, we will also add more EQs for CQs.
mana_ib is served by mana ethernet for RAW QPs and by RNIC for RC QPs. If RNIC is not available in the HW, we do not fail mana_ib and keep only RAW QP support. RNIC is availale only for port 1.
As a minimal usage, this patch series brings adding and removing RoCEv2 GIDs. For this, we set master netdev to the ib device and set required port parameters to get GIDs. RNIC of mana supports IPv6 and IPv4 addresses that are stored in the HW.
v1->v2:
* Fixed rcu_read_unlock() and updated commit message in "Enable RoCE on port 1"
Konstantin Taranov (5):
RDMA/mana_ib: Add EQ creation for rnic adapter
RDMA/mana_ib: Create and destroy rnic adapter
RDMA/mana_ib: Implement port parameters
RDMA/mana_ib: Enable RoCE on port 1
RDMA/mana_ib: Adding and deleting GIDs
drivers/infiniband/hw/mana/device.c | 28 ++++- drivers/infiniband/hw/mana/main.c | 203 ++++++++++++++++++++++++++++++++++- drivers/infiniband/hw/mana/mana_ib.h | 75 +++++++++++++ 3 files changed, 298 insertions(+), 8 deletions(-)
Signature · 1 line
1.8.3.1
PATCH v2 1/5 RDMA/mana_ib: Add EQ creation for rnic adapter
This patch introduces functions for RNIC creation
and creates one EQ for RNIC creation.
--- drivers/infiniband/hw/mana/device.c | 9 ++++-- drivers/infiniband/hw/mana/main.c | 53 ++++++++++++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 5 ++++ 3 files changed, 64 insertions(+), 3 deletions(-)
drivers/infiniband/hw/mana/device.c +6 -3
@@ -92,15 +92,19 @@ static int mana_ib_probe(struct auxiliary_device *adev, goto deregister_device; } +mana_ib_gd_create_rnic_adapter (dev); + ret =ib_register_device (&dev->ib_dev,"mana_%d" , mdev->gdma_context->dev);if (ret) - goto deregister_device; + goto destroy_rnic_adapter;dev_set_drvdata (&adev->dev, dev);return 0 ; +destroy_rnic_adapter: +mana_ib_gd_destroy_rnic_adapter (dev); deregister_device:mana_gd_deregister_device (dev->gdma_dev); free_ib_device:
@@ -113,9 +117,8 @@ static void mana_ib_remove(struct auxiliary_device *adev)struct mana_ib_dev *dev =dev_get_drvdata (&adev->dev);ib_unregister_device (&dev->ib_dev); - +mana_ib_gd_destroy_rnic_adapter (dev);mana_gd_deregister_device (dev->gdma_dev); -ib_dealloc_device (&dev->ib_dev); }
drivers/infiniband/hw/mana/main.c +53 -0
56 lines not on this page · raw message
drivers/infiniband/hw/mana/mana_ib.h +5 -0
@@ -48,6 +48,7 @@ struct mana_ib_adapter_caps {struct mana_ib_dev {struct ib_device ib_dev;struct gdma_dev *gdma_dev; +struct gdma_queue *fatal_err_eq;struct mana_ib_adapter_caps adapter_caps; };
@@ -228,4 +229,8 @@ int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,void mana_ib_disassociate_ucontext (struct ib_ucontext *ibcontext);int mana_ib_gd_query_adapter_caps (struct mana_ib_dev *mdev); + +void mana_ib_gd_create_rnic_adapter (struct mana_ib_dev *mdev); + +void mana_ib_gd_destroy_rnic_adapter (struct mana_ib_dev *mdev);#endif
Signature · 1 line
1.8.3.1
Re: [PATCH rdma-next v2 1/5] RDMA/mana_ib: Add EQ creation for rnic adapter
Quoted 2 lines · Konstantin Taranov
This patch introduces functions for RNIC creation
and creates one EQ for RNIC creation.
Please invest more time in commit messages, it is obvious that this patch "introduces functions for RNIC creation"" by looking in the code.
Quoted 71 lines
Signed-off-by Konstantin Taranov <[email protected]>--- drivers/infiniband/hw/mana/device.c | 9 ++++-- drivers/infiniband/hw/mana/main.c | 53 ++++++++++++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 5 ++++ 3 files changed, 64 insertions(+), 3 deletions(-)drivers/infiniband/hw/mana/device.c +6
-3@@ -92,15 +92,19 @@ static int mana_ib_probe(struct auxiliary_device *adev, goto deregister_device; } +mana_ib_gd_create_rnic_adapter (dev); + ret =ib_register_device (&dev->ib_dev,"mana_%d" , mdev->gdma_context->dev);if (ret) - goto deregister_device; + goto destroy_rnic_adapter;dev_set_drvdata (&adev->dev, dev);return 0 ; +destroy_rnic_adapter: +mana_ib_gd_destroy_rnic_adapter (dev); deregister_device:mana_gd_deregister_device (dev->gdma_dev); free_ib_device:@@ -113,9 +117,8 @@ static void mana_ib_remove(struct auxiliary_device *adev)struct mana_ib_dev *dev =dev_get_drvdata (&adev->dev);ib_unregister_device (&dev->ib_dev); - +mana_ib_gd_destroy_rnic_adapter (dev);mana_gd_deregister_device (dev->gdma_dev); -ib_dealloc_device (&dev->ib_dev); }drivers/infiniband/hw/mana/main.c +19
-0@@ -548,3 +548,56 @@ int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)return 0 ; } + +static int mana_ib_create_eqs (struct mana_ib_dev *mdev) +{ +struct gdma_context *gc =mdev_to_gc (mdev); +struct gdma_queue_spec spec = {}; +int err; + + spec.type =GDMA_EQ ; + spec.monitor_avl_buf = false; + spec.queue_size =EQ_SIZE ; + spec.eq.callback =NULL ; + spec.eq.context = mdev; + spec.eq.log2_throttle_limit =LOG2_EQ_THROTTLE ; + spec.eq.msix_index =0 ; + + err =mana_gd_create_mana_eq (&gc->mana_ib, &spec, &mdev->fatal_err_eq); +if (err) +return err; +
drivers/infiniband/hw/mana/main.c int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev) · line 579
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +10
-0+return 0 ; +} + +static void mana_ib_destroy_eqs (struct mana_ib_dev *mdev) +{ +if (!mdev->fatal_err_eq) +return ; + +mana_gd_destroy_queue (mdev_to_gc (mdev), mdev->fatal_err_eq); + mdev->fatal_err_eq =NULL ;
Please don't set NULL to the pointers if they are not used anymore.
Quoted 8 lines
drivers/infiniband/hw/mana/main.c +8
-0+} + +void mana_ib_gd_create_rnic_adapter (struct mana_ib_dev *mdev) +{ +int err; + + err =mana_ib_create_eqs (mdev); +if (err) {
drivers/infiniband/hw/mana/main.c int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev) · line 597
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +10
-0+ibdev_err (&mdev->ib_dev,"Failed to create EQs for RNIC err %d" , err); + goto cleanup; + } + +return ; + +cleanup: +ibdev_warn (&mdev->ib_dev, +"RNIC is not available. Only RAW QPs are supported" ); +mana_ib_destroy_eqs (mdev);
If mana_ib_create_eqs() fails, you shouldn't call to mana_ib_destroy_eqs(). mana_ib_create_eqs() needs to clean everything when it fails.
Thanks
Quoted 30 lines
drivers/infiniband/hw/mana/main.c +6
-0+} + +void mana_ib_gd_destroy_rnic_adapter (struct mana_ib_dev *mdev) +{ +mana_ib_destroy_eqs (mdev); +}drivers/infiniband/hw/mana/mana_ib.h +5
-0@@ -48,6 +48,7 @@ struct mana_ib_adapter_caps {struct mana_ib_dev {struct ib_device ib_dev;struct gdma_dev *gdma_dev; +struct gdma_queue *fatal_err_eq;struct mana_ib_adapter_caps adapter_caps; };@@ -228,4 +229,8 @@ int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,void mana_ib_disassociate_ucontext (struct ib_ucontext *ibcontext);int mana_ib_gd_query_adapter_caps (struct mana_ib_dev *mdev); + +void mana_ib_gd_create_rnic_adapter (struct mana_ib_dev *mdev); + +void mana_ib_gd_destroy_rnic_adapter (struct mana_ib_dev *mdev);#endif Signature · 1 line
1.8.3.1
PATCH v2 2/5 RDMA/mana_ib: Create and destroy rnic adapter
In reply to Konstantin TaranovThis patch adds RNIC creation and destruction.
If creation of RNIC fails, we support only RAW QPs as they are served by ethernet driver.
--- drivers/infiniband/hw/mana/main.c | 31 +++++++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+)
drivers/infiniband/hw/mana/main.c +31 -0
@@ -581,14 +581,31 @@ static void mana_ib_destroy_eqs(struct mana_ib_dev *mdev)void mana_ib_gd_create_rnic_adapter (struct mana_ib_dev *mdev) { +struct mana_rnic_create_adapter_resp resp = {}; +struct mana_rnic_create_adapter_req req = {}; +struct gdma_context *gc =mdev_to_gc (mdev);int err; + mdev->adapter_handle =INVALID_MANA_HANDLE ; + err =mana_ib_create_eqs (mdev);if (err) {ibdev_err (&mdev->ib_dev,"Failed to create EQs for RNIC err %d" , err); goto cleanup; } +mana_gd_init_req_hdr (&req.hdr,MANA_IB_CREATE_ADAPTER ,sizeof (req),sizeof (resp)); + req.hdr.req.msg_version =GDMA_MESSAGE_V2 ; + req.hdr.dev_id = gc->mana_ib.dev_id; + req.notify_eq_id = mdev->fatal_err_eq->id; + + err =mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); +if (err) { +ibdev_err (&mdev->ib_dev,"Failed to create RNIC adapter err %d" , err); + goto cleanup; + } + mdev->adapter_handle = resp.adapter; +return ; cleanup:
@@ -599,5 +616,19 @@ void mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev)void mana_ib_gd_destroy_rnic_adapter (struct mana_ib_dev *mdev) { +struct mana_rnic_destroy_adapter_resp resp = {}; +struct mana_rnic_destroy_adapter_req req = {}; +struct gdma_context *gc; + +if (!rnic_is_enabled (mdev)) +return ; + + gc =mdev_to_gc (mdev); +mana_gd_init_req_hdr (&req.hdr,MANA_IB_DESTROY_ADAPTER ,sizeof (req),sizeof (resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + +mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); + mdev->adapter_handle =INVALID_MANA_HANDLE ;mana_ib_destroy_eqs (mdev); }
drivers/infiniband/hw/mana/mana_ib.h +29 -0
@@ -48,6 +48,7 @@ struct mana_ib_adapter_caps {struct mana_ib_dev {struct ib_device ib_dev;struct gdma_dev *gdma_dev; +mana_handle_t adapter_handle;struct gdma_queue *fatal_err_eq;struct mana_ib_adapter_caps adapter_caps; };
@@ -115,6 +116,8 @@ struct mana_ib_rwq_ind_table {enum mana_ib_command_code {MANA_IB_GET_ADAPTER_CAP =0x30001 , +MANA_IB_CREATE_ADAPTER =0x30002 , +MANA_IB_DESTROY_ADAPTER =0x30003 , };struct mana_ib_query_adapter_caps_req {
@@ -143,6 +146,32 @@ struct mana_ib_query_adapter_caps_resp {u32 max_inline_data_size; };/* HW Data */ +struct mana_rnic_create_adapter_req { +struct gdma_req_hdr hdr; +u32 notify_eq_id; +u32 reserved; +u64 feature_flags; +};/*HW Data */ + +struct mana_rnic_create_adapter_resp { +struct gdma_resp_hdr hdr; +mana_handle_t adapter; +};/* HW Data */ + +struct mana_rnic_destroy_adapter_req { +struct gdma_req_hdr hdr; +mana_handle_t adapter; +};/*HW Data */ + +struct mana_rnic_destroy_adapter_resp { +struct gdma_resp_hdr hdr; +};/* HW Data */ + +static inline bool rnic_is_enabled (struct mana_ib_dev *mdev) +{ +return mdev->adapter_handle !=INVALID_MANA_HANDLE ; +} +static inline struct gdma_context *mdev_to_gc (struct mana_ib_dev *mdev) {return mdev->gdma_dev->gdma_context;
Signature · 1 line
1.8.3.1
Re: [PATCH rdma-next v2 2/5] RDMA/mana_ib: Create and destroy rnic adapter
Quoted 3 lines · Konstantin Taranov
This patch adds RNIC creation and destruction.
If creation of RNIC fails, we support only RAW QPs as they are served by ethernet driver.
So please make sure that you are creating RNIC only when you are supporting it. The idea that some function tries-and-fails with dmesg errors is not good idea.
Thanks
Quoted 120 lines
Signed-off-by Konstantin Taranov <[email protected]>--- drivers/infiniband/hw/mana/main.c | 31 +++++++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+)drivers/infiniband/hw/mana/main.c +31
-0@@ -581,14 +581,31 @@ static void mana_ib_destroy_eqs(struct mana_ib_dev *mdev)void mana_ib_gd_create_rnic_adapter (struct mana_ib_dev *mdev) { +struct mana_rnic_create_adapter_resp resp = {}; +struct mana_rnic_create_adapter_req req = {}; +struct gdma_context *gc =mdev_to_gc (mdev);int err; + mdev->adapter_handle =INVALID_MANA_HANDLE ; + err =mana_ib_create_eqs (mdev);if (err) {ibdev_err (&mdev->ib_dev,"Failed to create EQs for RNIC err %d" , err); goto cleanup; } +mana_gd_init_req_hdr (&req.hdr,MANA_IB_CREATE_ADAPTER ,sizeof (req),sizeof (resp)); + req.hdr.req.msg_version =GDMA_MESSAGE_V2 ; + req.hdr.dev_id = gc->mana_ib.dev_id; + req.notify_eq_id = mdev->fatal_err_eq->id; + + err =mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); +if (err) { +ibdev_err (&mdev->ib_dev,"Failed to create RNIC adapter err %d" , err); + goto cleanup; + } + mdev->adapter_handle = resp.adapter; +return ; cleanup:@@ -599,5 +616,19 @@ void mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev)void mana_ib_gd_destroy_rnic_adapter (struct mana_ib_dev *mdev) { +struct mana_rnic_destroy_adapter_resp resp = {}; +struct mana_rnic_destroy_adapter_req req = {}; +struct gdma_context *gc; + +if (!rnic_is_enabled (mdev)) +return ; + + gc =mdev_to_gc (mdev); +mana_gd_init_req_hdr (&req.hdr,MANA_IB_DESTROY_ADAPTER ,sizeof (req),sizeof (resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + +mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); + mdev->adapter_handle =INVALID_MANA_HANDLE ;mana_ib_destroy_eqs (mdev); }drivers/infiniband/hw/mana/mana_ib.h +29
-0@@ -48,6 +48,7 @@ struct mana_ib_adapter_caps {struct mana_ib_dev {struct ib_device ib_dev;struct gdma_dev *gdma_dev; +mana_handle_t adapter_handle;struct gdma_queue *fatal_err_eq;struct mana_ib_adapter_caps adapter_caps; };@@ -115,6 +116,8 @@ struct mana_ib_rwq_ind_table {enum mana_ib_command_code {MANA_IB_GET_ADAPTER_CAP =0x30001 , +MANA_IB_CREATE_ADAPTER =0x30002 , +MANA_IB_DESTROY_ADAPTER =0x30003 , };struct mana_ib_query_adapter_caps_req {@@ -143,6 +146,32 @@ struct mana_ib_query_adapter_caps_resp {u32 max_inline_data_size; };/* HW Data */ +struct mana_rnic_create_adapter_req { +struct gdma_req_hdr hdr; +u32 notify_eq_id; +u32 reserved; +u64 feature_flags; +};/*HW Data */ + +struct mana_rnic_create_adapter_resp { +struct gdma_resp_hdr hdr; +mana_handle_t adapter; +};/* HW Data */ + +struct mana_rnic_destroy_adapter_req { +struct gdma_req_hdr hdr; +mana_handle_t adapter; +};/*HW Data */ + +struct mana_rnic_destroy_adapter_resp { +struct gdma_resp_hdr hdr; +};/* HW Data */ + +static inline bool rnic_is_enabled (struct mana_ib_dev *mdev) +{ +return mdev->adapter_handle !=INVALID_MANA_HANDLE ; +} +static inline struct gdma_context *mdev_to_gc (struct mana_ib_dev *mdev) {return mdev->gdma_dev->gdma_context;Signature · 1 line
1.8.3.1
Konstantin Taranov
maintainerRE: [EXTERNAL] Re: [PATCH rdma-next v2 2/5] RDMA/mana_ib: Create and destroy rnic adapter
Quoted 6 lines
From: Leon Romanovsky <[email protected]>
Quoted 3 lines · Konstantin Taranov
This patch adds RNIC creation and destruction.
If creation of RNIC fails, we support only RAW QPs as they are served by ethernet driver.
Quoted 6 lines
So please make sure that you are creating RNIC only when you are supporting it. The idea that some function tries-and-fails with dmesg errors is not good idea.
Thanks
Hi Leon. Thanks for your comments and suggestion. I will incorporate them in the next version. Regarding this "try-and-fail", we cannot guarantee now that RNIC is supported, and try-and-fail is the only way to skip RNIC creation without impeding RAW QPs. Could you, please, suggest how we could correctly incorporate the "try-and-fail" strategy to get it upstreamed?
Quoted 124 lines
Quoted 4 lines
Signed-off-by Konstantin Taranov <[email protected]>--- drivers/infiniband/hw/mana/main.c | 31+++++++++++++++++++++++++++++++
Quoted 13 lines
drivers/infiniband/hw/mana/mana_ib.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+)drivers/infiniband/hw/mana/main.c +0
-0@@ -581,14 +581,31 @@ static void mana_ib_destroy_eqs(struct mana_ib_dev *mdev)void mana_ib_gd_create_rnic_adapter (struct mana_ib_dev *mdev) {drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct · line 593
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +5
-0+struct mana_rnic_create_adapter_resp resp = {}; +struct mana_rnic_create_adapter_req req = {}; +struct gdma_context *gc =mdev_to_gc (mdev);int err; + mdev->adapter_handle =INVALID_MANA_HANDLE ; + err =mana_ib_create_eqs (mdev);if (err) { ibdev_err(&mdev->ib_dev, "Failed to create EQs for RNIC err %d",err);
drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct mana_ib_dev *mdev)
Quoted 4 lines
drivers/infiniband/hw/mana/main.c +1
-0goto cleanup; } + mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_ADAPTER,sizeof(req), sizeof(resp));
drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct mana_ib_dev *mdev)
Quoted 7 lines
drivers/infiniband/hw/mana/main.c +7
-0+ req.hdr.req.msg_version =GDMA_MESSAGE_V2 ; + req.hdr.dev_id = gc->mana_ib.dev_id; + req.notify_eq_id = mdev->fatal_err_eq->id; + + err =mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); +if (err) { + ibdev_err(&mdev->ib_dev, "Failed to create RNIC adapter err %d",err);
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +4
-0+ goto cleanup; + } + mdev->adapter_handle = resp.adapter; +return ; cleanup:@@ -599,5 +616,19 @@ void mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev)drivers/infiniband/hw/mana/main.c void mana_ib_gd_create_rnic_adapter(struct · line 627
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +9
-0void mana_ib_gd_destroy_rnic_adapter (struct mana_ib_dev *mdev) { +struct mana_rnic_destroy_adapter_resp resp = {}; +struct mana_rnic_destroy_adapter_req req = {}; +struct gdma_context *gc; + +if (!rnic_is_enabled (mdev)) +return ; + + gc =mdev_to_gc (mdev); + mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_ADAPTER,sizeof(req), sizeof(resp));
Quoted 61 lines
drivers/infiniband/hw/mana/main.c +5
-0+ req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + +mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); + mdev->adapter_handle =INVALID_MANA_HANDLE ;mana_ib_destroy_eqs (mdev); }drivers/infiniband/hw/mana/mana_ib.h +1
-0@@ -48,6 +48,7 @@ struct mana_ib_adapter_caps { struct mana_ib_dev {struct ib_device ib_dev;struct gdma_dev *gdma_dev; +mana_handle_t adapter_handle;struct gdma_queue *fatal_err_eq; struct mana_ib_adapter_caps adapter_caps; }; @@ -115,6 +116,8 @@ struct mana_ib_rwq_ind_table {enum mana_ib_command_code { MANA_IB_GET_ADAPTER_CAP = 0x30001,drivers/infiniband/hw/mana/mana_ib.h +2
-0+MANA_IB_CREATE_ADAPTER =0x30002 , + MANA_IB_DESTROY_ADAPTER = 0x30003, }; struct mana_ib_query_adapter_caps_req { @@ -143,6 +146,32 @@ structmana_ib_query_adapter_caps_resp { u32 max_inline_data_size;}; /* HW Data */
drivers/infiniband/hw/mana/mana_ib.h +24
-0+struct mana_rnic_create_adapter_req { +struct gdma_req_hdr hdr; +u32 notify_eq_id; +u32 reserved; +u64 feature_flags; +};/*HW Data */ + +struct mana_rnic_create_adapter_resp { +struct gdma_resp_hdr hdr; +mana_handle_t adapter; +};/* HW Data */ + +struct mana_rnic_destroy_adapter_req { +struct gdma_req_hdr hdr; +mana_handle_t adapter; +};/*HW Data */ + +struct mana_rnic_destroy_adapter_resp { +struct gdma_resp_hdr hdr; +};/* HW Data */ + +static inline bool rnic_is_enabled (struct mana_ib_dev *mdev) { +return mdev->adapter_handle !=INVALID_MANA_HANDLE ; } + static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev*mdev) { return mdev->gdma_dev->gdma_context;--
1.8.3.1
Re: [EXTERNAL] Re: [PATCH rdma-next v2 2/5] RDMA/mana_ib: Create and destroy rnic adapter
Quoted 10 lines · Konstantin Taranov
Quoted 10 lines
From: Leon Romanovsky <[email protected]>
Quoted 3 lines · Konstantin Taranov
This patch adds RNIC creation and destruction.
If creation of RNIC fails, we support only RAW QPs as they are served by ethernet driver.So please make sure that you are creating RNIC only when you are supporting it. The idea that some function tries-and-fails with dmesg errors is not good idea.
Quoted 7 lines
Quoted 2 lines
Thanks
Hi Leon. Thanks for your comments and suggestion. I will incorporate them in the next version. Regarding this "try-and-fail", we cannot guarantee now that RNIC is supported, and try-and-fail is the only way to skip RNIC creation without impeding RAW QPs. Could you, please, suggest how we could correctly incorporate the "try-and-fail" strategy to get it upstreamed?
You already query NIC for its capabilities, so you can check if it supports RNIC.
Quoted 125 lines
Quoted 125 lines
Quoted 5 lines
Signed-off-by Konstantin Taranov <[email protected]>--- drivers/infiniband/hw/mana/main.c | 31+++++++++++++++++++++++++++++++
Quoted 13 lines
drivers/infiniband/hw/mana/mana_ib.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+)drivers/infiniband/hw/mana/main.c +0
-0@@ -581,14 +581,31 @@ static void mana_ib_destroy_eqs(struct mana_ib_dev *mdev)void mana_ib_gd_create_rnic_adapter (struct mana_ib_dev *mdev) {drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct · line 593
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +5
-0+struct mana_rnic_create_adapter_resp resp = {}; +struct mana_rnic_create_adapter_req req = {}; +struct gdma_context *gc =mdev_to_gc (mdev);int err; + mdev->adapter_handle =INVALID_MANA_HANDLE ; + err =mana_ib_create_eqs (mdev);if (err) { ibdev_err(&mdev->ib_dev, "Failed to create EQs for RNIC err %d",err);
drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct mana_ib_dev *mdev)
Quoted 4 lines
drivers/infiniband/hw/mana/main.c +1
-0goto cleanup; } + mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_ADAPTER,sizeof(req), sizeof(resp));
drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct mana_ib_dev *mdev)
Quoted 7 lines
drivers/infiniband/hw/mana/main.c +7
-0+ req.hdr.req.msg_version =GDMA_MESSAGE_V2 ; + req.hdr.dev_id = gc->mana_ib.dev_id; + req.notify_eq_id = mdev->fatal_err_eq->id; + + err =mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); +if (err) { + ibdev_err(&mdev->ib_dev, "Failed to create RNIC adapter err %d",err);
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +4
-0+ goto cleanup; + } + mdev->adapter_handle = resp.adapter; +return ; cleanup:@@ -599,5 +616,19 @@ void mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev)drivers/infiniband/hw/mana/main.c void mana_ib_gd_create_rnic_adapter(struct · line 627
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +9
-0void mana_ib_gd_destroy_rnic_adapter (struct mana_ib_dev *mdev) { +struct mana_rnic_destroy_adapter_resp resp = {}; +struct mana_rnic_destroy_adapter_req req = {}; +struct gdma_context *gc; + +if (!rnic_is_enabled (mdev)) +return ; + + gc =mdev_to_gc (mdev); + mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_ADAPTER,sizeof(req), sizeof(resp));
Quoted 61 lines
drivers/infiniband/hw/mana/main.c +5
-0+ req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + +mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); + mdev->adapter_handle =INVALID_MANA_HANDLE ;mana_ib_destroy_eqs (mdev); }drivers/infiniband/hw/mana/mana_ib.h +1
-0@@ -48,6 +48,7 @@ struct mana_ib_adapter_caps { struct mana_ib_dev {struct ib_device ib_dev;struct gdma_dev *gdma_dev; +mana_handle_t adapter_handle;struct gdma_queue *fatal_err_eq; struct mana_ib_adapter_caps adapter_caps; }; @@ -115,6 +116,8 @@ struct mana_ib_rwq_ind_table {enum mana_ib_command_code { MANA_IB_GET_ADAPTER_CAP = 0x30001,drivers/infiniband/hw/mana/mana_ib.h +2
-0+MANA_IB_CREATE_ADAPTER =0x30002 , + MANA_IB_DESTROY_ADAPTER = 0x30003, }; struct mana_ib_query_adapter_caps_req { @@ -143,6 +146,32 @@ structmana_ib_query_adapter_caps_resp { u32 max_inline_data_size;}; /* HW Data */
drivers/infiniband/hw/mana/mana_ib.h +24
-0+struct mana_rnic_create_adapter_req { +struct gdma_req_hdr hdr; +u32 notify_eq_id; +u32 reserved; +u64 feature_flags; +};/*HW Data */ + +struct mana_rnic_create_adapter_resp { +struct gdma_resp_hdr hdr; +mana_handle_t adapter; +};/* HW Data */ + +struct mana_rnic_destroy_adapter_req { +struct gdma_req_hdr hdr; +mana_handle_t adapter; +};/*HW Data */ + +struct mana_rnic_destroy_adapter_resp { +struct gdma_resp_hdr hdr; +};/* HW Data */ + +static inline bool rnic_is_enabled (struct mana_ib_dev *mdev) { +return mdev->adapter_handle !=INVALID_MANA_HANDLE ; } + static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev*mdev) { return mdev->gdma_dev->gdma_context;--
1.8.3.1
Konstantin Taranov
maintainerRE: [EXTERNAL] Re: [PATCH rdma-next v2 2/5] RDMA/mana_ib: Create and destroy rnic adapter
Quoted 15 lines
From: Leon Romanovsky <[email protected]>
Quoted 13 lines · Konstantin Taranov
Quoted 12 lines
From: Leon Romanovsky <[email protected]> On Fri, Feb 02, 2024 at
Quoted 3 lines · Konstantin Taranov
This patch adds RNIC creation and destruction.
If creation of RNIC fails, we support only RAW QPs as they are served by ethernet driver.So please make sure that you are creating RNIC only when you are supporting it. The idea that some function tries-and-fails with dmesg errors is not good idea.
Thanks
Quoted 8 lines
Quoted 1 line
Hi Leon. Thanks for your comments and suggestion. I will incorporate them
in the next version.
Quoted 3 lines
Regarding this "try-and-fail", we cannot guarantee now that RNIC is supported, and try-and-fail is the only way to skip RNIC creation without impeding RAW QPs. Could you, please, suggest how we could
correctly incorporate the "try-and-fail" strategy to get it upstreamed?
You already query NIC for its capabilities, so you can check if it supports RNIC.
At the moment, the capabilities do not indicate whether RNIC creation will be successful.
The reason is additional checks during RNIC creation that are not reflected in capabilities. The question is whether we can have the proposed "try and disable" or we must opt for failing the whole mana_ib.
Quoted 130 lines
Quoted 35 lines
Quoted 35 lines
Quoted 6 lines
Signed-off-by Konstantin Taranov <[email protected]>--- drivers/infiniband/hw/mana/main.c | 31+++++++++++++++++++++++++++++++
Quoted 14 lines
drivers/infiniband/hw/mana/mana_ib.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+)drivers/infiniband/hw/mana/main.c +1
-0@@ -581,14 +581,31 @@ static void mana_ib_destroy_eqs(struct mana_ib_dev *mdev)void mana_ib_gd_create_rnic_adapter (struct mana_ib_dev *mdev) { +struct mana_rnic_create_adapter_resp resp = {};drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct · line 594
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +4
-0+struct mana_rnic_create_adapter_req req = {}; +struct gdma_context *gc =mdev_to_gc (mdev);int err; + mdev->adapter_handle =INVALID_MANA_HANDLE ; + err =mana_ib_create_eqs (mdev); if (err) { ibdev_err(&mdev->ib_dev, "Failed to create EQs for RNIC err %d",err);
Quoted 3 lines
drivers/infiniband/hw/mana/main.c +0
-0goto cleanup; }Quoted 7 lines
Quoted 7 lines
drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct · line 598
Quoted 1 line
drivers/infiniband/hw/mana/main.c +1
-0+ mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_ADAPTER,sizeof(req), sizeof(resp));
Quoted 5 lines
drivers/infiniband/hw/mana/main.c +5
-0+ req.hdr.req.msg_version =GDMA_MESSAGE_V2 ; + req.hdr.dev_id = gc->mana_ib.dev_id; + req.notify_eq_id = mdev->fatal_err_eq->id; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp),&resp);
Quoted 25 lines
Quoted 25 lines
drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct · line 606
Quoted 3 lines
drivers/infiniband/hw/mana/main.c +3
-0+ if (err) { + ibdev_err(&mdev->ib_dev, "Failed to create RNIC + adapter err %d",err);
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +4
-0+ goto cleanup; + } + mdev->adapter_handle = resp.adapter; +return ; cleanup:@@ -599,5 +616,19 @@ void mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev)drivers/infiniband/hw/mana/main.c void mana_ib_gd_create_rnic_adapter(struct · line 627
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +9
-0void mana_ib_gd_destroy_rnic_adapter (struct mana_ib_dev *mdev) { +struct mana_rnic_destroy_adapter_resp resp = {}; +struct mana_rnic_destroy_adapter_req req = {}; +struct gdma_context *gc; + +if (!rnic_is_enabled (mdev)) +return ; + + gc =mdev_to_gc (mdev); + mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_ADAPTER,sizeof(req), sizeof(resp));
Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
drivers/infiniband/hw/mana/main.c +5
-0+ req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + +mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); + mdev->adapter_handle =INVALID_MANA_HANDLE ;Quoted 8 lines
drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_adapter_caps { struct
Quoted 8 lines
drivers/infiniband/hw/mana/main.c +0
-0mana_ib_destroy_eqs (mdev); }drivers/infiniband/hw/mana/mana_ib.h +0
-0@@ -48,6 +48,7 @@ struct mana_ib_adapter_caps { struct
mana_ib_dev{Quoted 48 lines
Quoted 48 lines
Quoted 48 lines
drivers/infiniband/hw/mana/mana_ib.h +4
-0struct ib_device ib_dev;struct gdma_dev *gdma_dev; +mana_handle_t adapter_handle;struct gdma_queue *fatal_err_eq; struct mana_ib_adapter_caps adapter_caps; }; @@ -115,6 +116,8 @@ struct mana_ib_rwq_ind_table { enum mana_ib_command_code { MANA_IB_GET_ADAPTER_CAP = 0x30001, + MANA_IB_CREATE_ADAPTER = 0x30002, + MANA_IB_DESTROY_ADAPTER = 0x30003, }; struct mana_ib_query_adapter_caps_req { @@ -143,6 +146,32 @@struct mana_ib_query_adapter_caps_resp { u32 max_inline_data_size;}; /* HW Data */
drivers/infiniband/hw/mana/mana_ib.h +24
-0+struct mana_rnic_create_adapter_req { +struct gdma_req_hdr hdr; +u32 notify_eq_id; +u32 reserved; +u64 feature_flags; +};/*HW Data */ + +struct mana_rnic_create_adapter_resp { +struct gdma_resp_hdr hdr; +mana_handle_t adapter; +};/* HW Data */ + +struct mana_rnic_destroy_adapter_req { +struct gdma_req_hdr hdr; +mana_handle_t adapter; +};/*HW Data */ + +struct mana_rnic_destroy_adapter_resp { +struct gdma_resp_hdr hdr; +};/* HW Data */ + +static inline bool rnic_is_enabled (struct mana_ib_dev *mdev) { +return mdev->adapter_handle !=INVALID_MANA_HANDLE ; } + static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev*mdev) { return mdev->gdma_dev->gdma_context;--
1.8.3.1
Re: [EXTERNAL] Re: [PATCH rdma-next v2 2/5] RDMA/mana_ib: Create and destroy rnic adapter
Quoted 20 lines · Konstantin Taranov
Quoted 20 lines
From: Leon Romanovsky <[email protected]>
Quoted 6 lines · Konstantin Taranov
Quoted 6 lines
From: Leon Romanovsky <[email protected]> On Fri, Feb 02, 2024 at
Quoted 3 lines · Konstantin Taranov
This patch adds RNIC creation and destruction.
If creation of RNIC fails, we support only RAW QPs as they are served by ethernet driver.Quoted 8 lines
Quoted 6 lines
So please make sure that you are creating RNIC only when you are supporting it. The idea that some function tries-and-fails with dmesg errors is not good idea.
Thanks
Hi Leon. Thanks for your comments and suggestion. I will incorporate them
in the next version.
Quoted 3 lines
Regarding this "try-and-fail", we cannot guarantee now that RNIC is supported, and try-and-fail is the only way to skip RNIC creation without impeding RAW QPs. Could you, please, suggest how we could
Quoted 7 lines
Quoted 3 lines
correctly incorporate the "try-and-fail" strategy to get it upstreamed?
You already query NIC for its capabilities, so you can check if it supports RNIC.
At the moment, the capabilities do not indicate whether RNIC creation will be successful.
The reason is additional checks during RNIC creation that are not reflected in capabilities. The question is whether we can have the proposed "try and disable" or we must opt for failing the wholemana_ib.
RNIC creation can be seen as an example of any other feature which will be added later, you will never know if it will be successful or not without capabilities.
If you continue with this try-and-fail approach, I afraid that you will end up with whole driver written in this style. Style where you don't separate between "real" failures (wrong configuration, OOM e.t.c) and "expected" failures (feature is not supported).
Thanks
Quoted 131 lines
Quoted 131 lines
Quoted 36 lines
Quoted 36 lines
Quoted 7 lines
Signed-off-by Konstantin Taranov <[email protected]>--- drivers/infiniband/hw/mana/main.c | 31+++++++++++++++++++++++++++++++
Quoted 14 lines
drivers/infiniband/hw/mana/mana_ib.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+)drivers/infiniband/hw/mana/main.c +1
-0@@ -581,14 +581,31 @@ static void mana_ib_destroy_eqs(struct mana_ib_dev *mdev)void mana_ib_gd_create_rnic_adapter (struct mana_ib_dev *mdev) { +struct mana_rnic_create_adapter_resp resp = {};drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct · line 594
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +4
-0+struct mana_rnic_create_adapter_req req = {}; +struct gdma_context *gc =mdev_to_gc (mdev);int err; + mdev->adapter_handle =INVALID_MANA_HANDLE ; + err =mana_ib_create_eqs (mdev); if (err) { ibdev_err(&mdev->ib_dev, "Failed to create EQs for RNIC err %d",err);
Quoted 3 lines
drivers/infiniband/hw/mana/main.c +0
-0goto cleanup; }Quoted 7 lines
Quoted 7 lines
drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct · line 598
Quoted 1 line
drivers/infiniband/hw/mana/main.c +1
-0+ mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_ADAPTER,sizeof(req), sizeof(resp));
Quoted 5 lines
drivers/infiniband/hw/mana/main.c +5
-0+ req.hdr.req.msg_version =GDMA_MESSAGE_V2 ; + req.hdr.dev_id = gc->mana_ib.dev_id; + req.notify_eq_id = mdev->fatal_err_eq->id; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp),&resp);
Quoted 25 lines
Quoted 25 lines
drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct · line 606
Quoted 3 lines
drivers/infiniband/hw/mana/main.c +3
-0+ if (err) { + ibdev_err(&mdev->ib_dev, "Failed to create RNIC + adapter err %d",err);
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +4
-0+ goto cleanup; + } + mdev->adapter_handle = resp.adapter; +return ; cleanup:@@ -599,5 +616,19 @@ void mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev)drivers/infiniband/hw/mana/main.c void mana_ib_gd_create_rnic_adapter(struct · line 627
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +9
-0void mana_ib_gd_destroy_rnic_adapter (struct mana_ib_dev *mdev) { +struct mana_rnic_destroy_adapter_resp resp = {}; +struct mana_rnic_destroy_adapter_req req = {}; +struct gdma_context *gc; + +if (!rnic_is_enabled (mdev)) +return ; + + gc =mdev_to_gc (mdev); + mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_ADAPTER,sizeof(req), sizeof(resp));
Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
drivers/infiniband/hw/mana/main.c +5
-0+ req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + +mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); + mdev->adapter_handle =INVALID_MANA_HANDLE ;Quoted 8 lines
drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_adapter_caps { struct
Quoted 8 lines
drivers/infiniband/hw/mana/main.c +0
-0mana_ib_destroy_eqs (mdev); }drivers/infiniband/hw/mana/mana_ib.h +0
-0@@ -48,6 +48,7 @@ struct mana_ib_adapter_caps { struct
mana_ib_dev{Quoted 48 lines
Quoted 48 lines
Quoted 48 lines
drivers/infiniband/hw/mana/mana_ib.h +4
-0struct ib_device ib_dev;struct gdma_dev *gdma_dev; +mana_handle_t adapter_handle;struct gdma_queue *fatal_err_eq; struct mana_ib_adapter_caps adapter_caps; }; @@ -115,6 +116,8 @@ struct mana_ib_rwq_ind_table { enum mana_ib_command_code { MANA_IB_GET_ADAPTER_CAP = 0x30001, + MANA_IB_CREATE_ADAPTER = 0x30002, + MANA_IB_DESTROY_ADAPTER = 0x30003, }; struct mana_ib_query_adapter_caps_req { @@ -143,6 +146,32 @@struct mana_ib_query_adapter_caps_resp { u32 max_inline_data_size;}; /* HW Data */
drivers/infiniband/hw/mana/mana_ib.h +24
-0+struct mana_rnic_create_adapter_req { +struct gdma_req_hdr hdr; +u32 notify_eq_id; +u32 reserved; +u64 feature_flags; +};/*HW Data */ + +struct mana_rnic_create_adapter_resp { +struct gdma_resp_hdr hdr; +mana_handle_t adapter; +};/* HW Data */ + +struct mana_rnic_destroy_adapter_req { +struct gdma_req_hdr hdr; +mana_handle_t adapter; +};/*HW Data */ + +struct mana_rnic_destroy_adapter_resp { +struct gdma_resp_hdr hdr; +};/* HW Data */ + +static inline bool rnic_is_enabled (struct mana_ib_dev *mdev) { +return mdev->adapter_handle !=INVALID_MANA_HANDLE ; } + static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev*mdev) { return mdev->gdma_dev->gdma_context;--
1.8.3.1
Konstantin Taranov
maintainerRE: [EXTERNAL] Re: [PATCH rdma-next v2 2/5] RDMA/mana_ib: Create and destroy rnic adapter
Quoted 36 lines
From: Leon Romanovsky <[email protected]>
Quoted 17 lines · Konstantin Taranov
Quoted 17 lines
From: Leon Romanovsky <[email protected]> On Sun, Feb 04, 2024 at
Quoted 15 lines · Konstantin Taranov
Quoted 6 lines
From: Leon Romanovsky <[email protected]> On Fri, Feb 02, 2024 at
Quoted 3 lines · Konstantin Taranov
This patch adds RNIC creation and destruction.
If creation of RNIC fails, we support only RAW QPs as they are served by ethernet driver.Quoted 6 lines
So please make sure that you are creating RNIC only when you are supporting it. The idea that some function tries-and-fails with dmesg errors is not good idea.
Thanks
Hi Leon. Thanks for your comments and suggestion. I will incorporate them
Quoted 8 lines
Quoted 8 lines
in the next version.
Quoted 4 lines
Regarding this "try-and-fail", we cannot guarantee now that RNIC is supported, and try-and-fail is the only way to skip RNIC creation without impeding RAW QPs. Could you, please, suggest how we could
correctly incorporate the "try-and-fail" strategy to get it upstreamed?
You already query NIC for its capabilities, so you can check if it supports
RNIC.
Quoted 2 lines
At the moment, the capabilities do not indicate whether RNIC creation will
be successful.
Quoted 1 line
The reason is additional checks during RNIC creation that are not reflected
in capabilities.
Quoted 1 line
The question is whether we can have the proposed "try and disable" or we
must opt for failing the whole
mana_ib.
Quoted 9 lines
RNIC creation can be seen as an example of any other feature which will be added later, you will never know if it will be successful or not without capabilities.
If you continue with this try-and-fail approach, I afraid that you will end up with whole driver written in this style. Style where you don't separate between "real" failures (wrong configuration, OOM
e.t.c) and "expected" failures (feature is not supported).
Hi Leon. I understand your concerns and I see how try-and-fail approach can go wrong.
I think you misunderstood the current HW limitation we have. We *do* distinguish between failures and this " try-and-fail " will be used once during initialization. As I mentioned above, our current HW capabilities cannot reflect whether RNIC is supported. Therefore, we must try to create it to understand whether it is really supported. So, if we succeed then the RNIC feature is supported and all RNIC-related operations will work. Otherwise, RNIC capability is not present and in this case, we just wanted to warn the user about it. If it concerns you, I can remove this warn message.
Given the provided explanation, I would appreciate if you wrote whether this approach of querying RNIC support could be accepted.
Thanks!
Quoted 138 lines
Thanks
Quoted 62 lines
Quoted 62 lines
Quoted 38 lines
Quoted 38 lines
Quoted 8 lines
Signed-off-by: Konstantin Taranov
<[email protected]>--- drivers/infiniband/hw/mana/main.c | 31+++++++++++++++++++++++++++++++
Quoted 15 lines
drivers/infiniband/hw/mana/mana_ib.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+)drivers/infiniband/hw/mana/main.c +1
-0@@ -581,14 +581,31 @@ static void mana_ib_destroy_eqs(struct mana_ib_dev *mdev)void mana_ib_gd_create_rnic_adapter (struct mana_ib_dev *mdev) { +struct mana_rnic_create_adapter_resp resp = {};drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct · line 595
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +4
-0+struct mana_rnic_create_adapter_req req = {}; +struct gdma_context *gc =mdev_to_gc (mdev);int err; + mdev->adapter_handle =INVALID_MANA_HANDLE ; + err =mana_ib_create_eqs (mdev); if (err) { ibdev_err(&mdev->ib_dev, "Failed to create EQs for RNIC err %d",err);
Quoted 3 lines
drivers/infiniband/hw/mana/main.c +0
-0goto cleanup; }Quoted 8 lines
Quoted 8 lines
drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct · line 599
Quoted 1 line
drivers/infiniband/hw/mana/main.c +1
-0+ mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_ADAPTER,sizeof(req), sizeof(resp));
Quoted 6 lines
drivers/infiniband/hw/mana/main.c +6
-0+ req.hdr.req.msg_version =GDMA_MESSAGE_V2 ; + req.hdr.dev_id = gc->mana_ib.dev_id; + req.notify_eq_id = mdev->fatal_err_eq->id; + + err = mana_gd_send_request(gc, sizeof(req), &req, + sizeof(resp),&resp);
Quoted 15 lines
Quoted 15 lines
drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct · line 608
Quoted 3 lines
drivers/infiniband/hw/mana/main.c +3
-0+ if (err) { + ibdev_err(&mdev->ib_dev, "Failed to create RNIC + adapter err %d",err);
Quoted 11 lines
drivers/infiniband/hw/mana/main.c +4
-0+ goto cleanup; + } + mdev->adapter_handle = resp.adapter; +return ; cleanup:@@ -599,5 +616,19 @@ voidmana_ib_gd_create_rnic_adapter(struct
mana_ib_dev*mdev)Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
drivers/infiniband/hw/mana/main.c void mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev)
Quoted 11 lines
void mana_ib_gd_destroy_rnic_adapter(struct
mana_ib_dev
*mdev) {drivers/infiniband/hw/mana/main.c +9
-0+struct mana_rnic_destroy_adapter_resp resp = {}; +struct mana_rnic_destroy_adapter_req req = {}; +struct gdma_context *gc; + +if (!rnic_is_enabled (mdev)) +return ; + + gc =mdev_to_gc (mdev); + mana_gd_init_req_hdr(&req.hdr,
MANA_IB_DESTROY_ADAPTER,Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
sizeof(req), sizeof(resp));
Quoted 4 lines
drivers/infiniband/hw/mana/main.c +4
-0+ req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + + mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp),&resp);
Quoted 56 lines
Quoted 56 lines
Quoted 8 lines
Quoted 8 lines
drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_adapter_caps { struct
Quoted 8 lines
+ mdev->adapter_handle = INVALID_MANA_HANDLE; mana_ib_destroy_eqs(mdev); } diff --git
a/drivers/infiniband/hw/mana/mana_ib.hb/drivers/infiniband/hw/mana/mana_ib.h
index a4b94ee..96454cf 100644drivers/infiniband/hw/mana/mana_ib.h +0
-0@@ -48,6 +48,7 @@ struct mana_ib_adapter_caps { struct
mana_ib_dev{Quoted 47 lines
Quoted 47 lines
Quoted 47 lines
drivers/infiniband/hw/mana/mana_ib.h +4
-0struct ib_device ib_dev;struct gdma_dev *gdma_dev; +mana_handle_t adapter_handle;struct gdma_queue *fatal_err_eq; struct mana_ib_adapter_caps adapter_caps; }; @@ -115,6 +116,8 @@ struct mana_ib_rwq_ind_table { enum mana_ib_command_code { MANA_IB_GET_ADAPTER_CAP = 0x30001, + MANA_IB_CREATE_ADAPTER = 0x30002, + MANA_IB_DESTROY_ADAPTER = 0x30003, }; struct mana_ib_query_adapter_caps_req { @@ -143,6 +146,32 @@struct mana_ib_query_adapter_caps_resp { u32 max_inline_data_size; }; /* HW Data */drivers/infiniband/hw/mana/mana_ib.h +23
-0+struct mana_rnic_create_adapter_req { +struct gdma_req_hdr hdr; +u32 notify_eq_id; +u32 reserved; +u64 feature_flags; +};/*HW Data */ + +struct mana_rnic_create_adapter_resp { +struct gdma_resp_hdr hdr; +mana_handle_t adapter; +};/* HW Data */ + +struct mana_rnic_destroy_adapter_req { +struct gdma_req_hdr hdr; +mana_handle_t adapter; +};/*HW Data */ + +struct mana_rnic_destroy_adapter_resp { +struct gdma_resp_hdr hdr; };/* HW Data */ + +static inline bool rnic_is_enabled (struct mana_ib_dev *mdev) { +return mdev->adapter_handle !=INVALID_MANA_HANDLE ; } + static inline struct gdma_context *mdev_to_gc(struct
mana_ib_dev*mdev) { return mdev->gdma_dev->gdma_context;--
1.8.3.1
Re: [EXTERNAL] Re: [PATCH rdma-next v2 2/5] RDMA/mana_ib: Create and destroy rnic adapter
Quoted 46 lines · Konstantin Taranov
Quoted 36 lines
From: Leon Romanovsky <[email protected]>
Quoted 17 lines · Konstantin Taranov
Quoted 17 lines
From: Leon Romanovsky <[email protected]> On Sun, Feb 04, 2024 at
Quoted 15 lines · Konstantin Taranov
Quoted 6 lines
From: Leon Romanovsky <[email protected]> On Fri, Feb 02, 2024 at
Quoted 3 lines · Konstantin Taranov
This patch adds RNIC creation and destruction.
If creation of RNIC fails, we support only RAW QPs as they are served by ethernet driver.Quoted 6 lines
So please make sure that you are creating RNIC only when you are supporting it. The idea that some function tries-and-fails with dmesg errors is not good idea.
Thanks
Hi Leon. Thanks for your comments and suggestion. I will incorporate them
Quoted 8 lines
Quoted 8 lines
in the next version.
Quoted 4 lines
Regarding this "try-and-fail", we cannot guarantee now that RNIC is supported, and try-and-fail is the only way to skip RNIC creation without impeding RAW QPs. Could you, please, suggest how we could
correctly incorporate the "try-and-fail" strategy to get it upstreamed?
You already query NIC for its capabilities, so you can check if it supports
RNIC.
Quoted 2 lines
At the moment, the capabilities do not indicate whether RNIC creation will
be successful.
Quoted 1 line
The reason is additional checks during RNIC creation that are not reflected
in capabilities.
Quoted 1 line
The question is whether we can have the proposed "try and disable" or we
must opt for failing the whole
mana_ib.Quoted 9 lines
RNIC creation can be seen as an example of any other feature which will be added later, you will never know if it will be successful or not without capabilities.
If you continue with this try-and-fail approach, I afraid that you will end up with whole driver written in this style. Style where you don't separate between "real" failures (wrong configuration, OOM
e.t.c) and "expected" failures (feature is not supported).
Quoted 3 lines
Hi Leon. I understand your concerns and I see how try-and-fail approach can go wrong. I think you misunderstood the current HW limitation we have. We *do* distinguish between failures
This is not what the code is doing, you are ignoring real errors.
The distinguish is usually done by checking the return value of the function after looking after specific error code returned by FW/HW.
Quoted 8 lines
and this " try-and-fail " will be used once during initialization. As I mentioned above,
our current HW capabilities cannot reflect whether RNIC is supported. Therefore, we must try to create it to understand whether it is really supported. So, if we succeed then the RNIC feature is supported and all RNIC-related operations will work. Otherwise, RNIC capability is not present and in this case, we just wanted to warn the user about it. If it concerns you, I can remove this warn message.Given the provided explanation, I would appreciate if you wrote whether this approach of querying RNIC support could be accepted.
Unless you have a good explanation why you can add new FW command to configure RNIC, but can't add FW command to query if RNIC is supported. I'm not keen on adopting this approach.
Thanks
Quoted 141 lines
Thanks!
Quoted 138 lines
Thanks
Quoted 62 lines
Quoted 62 lines
Quoted 38 lines
Quoted 38 lines
Quoted 8 lines
Signed-off-by: Konstantin Taranov
<[email protected]>--- drivers/infiniband/hw/mana/main.c | 31+++++++++++++++++++++++++++++++
Quoted 15 lines
drivers/infiniband/hw/mana/mana_ib.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+)drivers/infiniband/hw/mana/main.c +1
-0@@ -581,14 +581,31 @@ static void mana_ib_destroy_eqs(struct mana_ib_dev *mdev)void mana_ib_gd_create_rnic_adapter (struct mana_ib_dev *mdev) { +struct mana_rnic_create_adapter_resp resp = {};drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct · line 595
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +4
-0+struct mana_rnic_create_adapter_req req = {}; +struct gdma_context *gc =mdev_to_gc (mdev);int err; + mdev->adapter_handle =INVALID_MANA_HANDLE ; + err =mana_ib_create_eqs (mdev); if (err) { ibdev_err(&mdev->ib_dev, "Failed to create EQs for RNIC err %d",err);
Quoted 3 lines
drivers/infiniband/hw/mana/main.c +0
-0goto cleanup; }Quoted 8 lines
Quoted 8 lines
drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct · line 599
Quoted 1 line
drivers/infiniband/hw/mana/main.c +1
-0+ mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_ADAPTER,sizeof(req), sizeof(resp));
Quoted 6 lines
drivers/infiniband/hw/mana/main.c +6
-0+ req.hdr.req.msg_version =GDMA_MESSAGE_V2 ; + req.hdr.dev_id = gc->mana_ib.dev_id; + req.notify_eq_id = mdev->fatal_err_eq->id; + + err = mana_gd_send_request(gc, sizeof(req), &req, + sizeof(resp),&resp);
Quoted 15 lines
Quoted 15 lines
drivers/infiniband/hw/mana/main.c static void mana_ib_destroy_eqs(struct · line 608
Quoted 3 lines
drivers/infiniband/hw/mana/main.c +3
-0+ if (err) { + ibdev_err(&mdev->ib_dev, "Failed to create RNIC + adapter err %d",err);
Quoted 11 lines
drivers/infiniband/hw/mana/main.c +4
-0+ goto cleanup; + } + mdev->adapter_handle = resp.adapter; +return ; cleanup:@@ -599,5 +616,19 @@ voidmana_ib_gd_create_rnic_adapter(struct
mana_ib_dev*mdev)Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
drivers/infiniband/hw/mana/main.c
Quoted 11 lines
void mana_ib_gd_destroy_rnic_adapter(struct
mana_ib_dev
*mdev) {drivers/infiniband/hw/mana/main.c +9
-0+struct mana_rnic_destroy_adapter_resp resp = {}; +struct mana_rnic_destroy_adapter_req req = {}; +struct gdma_context *gc; + +if (!rnic_is_enabled (mdev)) +return ; + + gc =mdev_to_gc (mdev); + mana_gd_init_req_hdr(&req.hdr,
MANA_IB_DESTROY_ADAPTER,Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
sizeof(req), sizeof(resp));
Quoted 4 lines
drivers/infiniband/hw/mana/main.c +4
-0+ req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + + mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp),&resp);
Quoted 56 lines
Quoted 56 lines
Quoted 8 lines
Quoted 8 lines
drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_adapter_caps { struct
Quoted 8 lines
+ mdev->adapter_handle = INVALID_MANA_HANDLE; mana_ib_destroy_eqs(mdev); } diff --git
a/drivers/infiniband/hw/mana/mana_ib.hb/drivers/infiniband/hw/mana/mana_ib.h
index a4b94ee..96454cf 100644drivers/infiniband/hw/mana/mana_ib.h +0
-0@@ -48,6 +48,7 @@ struct mana_ib_adapter_caps { struct
mana_ib_dev{Quoted 47 lines
Quoted 47 lines
Quoted 47 lines
drivers/infiniband/hw/mana/mana_ib.h +4
-0struct ib_device ib_dev;struct gdma_dev *gdma_dev; +mana_handle_t adapter_handle;struct gdma_queue *fatal_err_eq; struct mana_ib_adapter_caps adapter_caps; }; @@ -115,6 +116,8 @@ struct mana_ib_rwq_ind_table { enum mana_ib_command_code { MANA_IB_GET_ADAPTER_CAP = 0x30001, + MANA_IB_CREATE_ADAPTER = 0x30002, + MANA_IB_DESTROY_ADAPTER = 0x30003, }; struct mana_ib_query_adapter_caps_req { @@ -143,6 +146,32 @@struct mana_ib_query_adapter_caps_resp { u32 max_inline_data_size; }; /* HW Data */drivers/infiniband/hw/mana/mana_ib.h +23
-0+struct mana_rnic_create_adapter_req { +struct gdma_req_hdr hdr; +u32 notify_eq_id; +u32 reserved; +u64 feature_flags; +};/*HW Data */ + +struct mana_rnic_create_adapter_resp { +struct gdma_resp_hdr hdr; +mana_handle_t adapter; +};/* HW Data */ + +struct mana_rnic_destroy_adapter_req { +struct gdma_req_hdr hdr; +mana_handle_t adapter; +};/*HW Data */ + +struct mana_rnic_destroy_adapter_resp { +struct gdma_resp_hdr hdr; };/* HW Data */ + +static inline bool rnic_is_enabled (struct mana_ib_dev *mdev) { +return mdev->adapter_handle !=INVALID_MANA_HANDLE ; } + static inline struct gdma_context *mdev_to_gc(struct
mana_ib_dev*mdev) { return mdev->gdma_dev->gdma_context;--
1.8.3.1
Konstantin Taranov
maintainerRe: [PATCH rdma-next v2 2/5] RDMA/mana_ib: Create and destroy rnic adapter
Quoted 81 lines
From: Leon Romanovsky <[email protected]>
Sent: Monday, 5 February 2024 10:57
To: Konstantin Taranov <[email protected]>
Cc: Konstantin Taranov <[email protected]>; [email protected]; Long Li <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: Re: [EXTERNAL] Re: [PATCH rdma-next v2 2/5] RDMA/mana_ib:
Create and destroy rnic adapter[You don't often get email from [email protected]. Learn why this is important at https://aka.ms/LearnAboutSenderIdentification ]
Quoted 40 lines · Konstantin Taranov
Quoted 40 lines
From: Leon Romanovsky <[email protected]> On Sun, Feb 04, 2024 at
Quoted 18 lines · Konstantin Taranov
Quoted 18 lines
From: Leon Romanovsky <[email protected]> On Sun, Feb 04, 2024 at
Quoted 10 lines · Konstantin Taranov
Quoted 10 lines
From: Leon Romanovsky <[email protected]> On Fri, Feb 02, 2024
Quoted 3 lines · Konstantin Taranov
This patch adds RNIC creation and destruction.
If creation of RNIC fails, we support only RAW QPs as they are served by ethernet driver.So please make sure that you are creating RNIC only when you are supporting it. The idea that some function tries-and-fails with dmesg errors is not good idea.
Quoted 5 lines
Quoted 2 lines
Thanks
Hi Leon. Thanks for your comments and suggestion. I will incorporate them
in the next version.
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines
Regarding this "try-and-fail", we cannot guarantee now that RNIC is supported, and try-and-fail is the only way to skip RNIC creation without impeding RAW QPs. Could you, please, suggest how we could
correctly incorporate the "try-and-fail" strategy to get it upstreamed?
You already query NIC for its capabilities, so you can check if it supports
RNIC.
Quoted 3 lines
At the moment, the capabilities do not indicate whether RNIC creation will
be successful.
Quoted 2 lines
The reason is additional checks during RNIC creation that are not reflected
in capabilities.
Quoted 2 lines
The question is whether we can have the proposed "try and disable" or we
must opt for failing the whole
mana_ib.Quoted 7 lines
Quoted 7 lines
RNIC creation can be seen as an example of any other feature which will be added later, you will never know if it will be successful or not without capabilities.
If you continue with this try-and-fail approach, I afraid that you will end up with whole driver written in this style. Style where you don't separate between "real" failures (wrong configuration, OOM
e.t.c)and "expected"
Quoted 4 lines
Quoted 2 lines
failures (feature is not supported).
Hi Leon. I understand your concerns and I see how try-and-fail approach can
go wrong.
Quoted 2 lines
I think you misunderstood the current HW limitation we have. We *do* distinguish between failures
This is not what the code is doing, you are ignoring real errors. The distinguish is usually done by checking the return value of the function after looking after specific error code returned by FW/HW.
Quoted 5 lines
and this " try-and-fail " will be used once during initialization. As I mentioned above, our current HW capabilities cannot reflect whether RNIC is supported. Therefore, we must try to create it to understand whether it is really supported. So, if we succeed then the RNIC feature is supported and all RNIC-related operations will work. Otherwise,
RNIC capability is not present and in this case, we just wanted to warn the user about it. If it concerns you, I can remove this warn message.
Quoted 1 line
Quoted 7 lines
Quoted 2 lines
Given the provided explanation, I would appreciate if you wrote whether this approach of querying RNIC support could be accepted.
Unless you have a good explanation why you can add new FW command to configure RNIC, but can't add FW command to query if RNIC is supported. I'm not keen on adopting this approach.
The main reason was backward compatibility with old firmware that had the aforementioned limitation. Anyway, we will try to internally retire the old firmware and will send the v3 patches without the "try and fail" approach (in 2-3 weeks).
Thanks.
Quoted 145 lines
Thanks
Quoted 143 lines
Thanks!
Quoted 140 lines
Thanks
Quoted 63 lines
Quoted 63 lines
Quoted 40 lines
Quoted 40 lines
Quoted 8 lines
Signed-off-by: Konstantin Taranov
<[email protected]>--- drivers/infiniband/hw/mana/main.c | 31+++++++++++++++++++++++++++++++
Quoted 15 lines
drivers/infiniband/hw/mana/mana_ib.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+)drivers/infiniband/hw/mana/main.c +1
-0@@ -581,14 +581,31 @@ static void mana_ib_destroy_eqs(struct mana_ib_dev *mdev) void mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev) { +struct mana_rnic_create_adapter_resp resp = {};drivers/infiniband/hw/mana/main.c static void · line 595
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +4
-0+struct mana_rnic_create_adapter_req req = {}; +struct gdma_context *gc =mdev_to_gc (mdev);int err; + mdev->adapter_handle =INVALID_MANA_HANDLE ; + err =mana_ib_create_eqs (mdev); if (err) { ibdev_err(&mdev->ib_dev, "Failed to create EQs for RNIC err %d",err);
Quoted 5 lines
drivers/infiniband/hw/mana/main.c +2
-0goto cleanup; } + mana_gd_init_req_hdr(&req.hdr, + MANA_IB_CREATE_ADAPTER,Quoted 7 lines
Quoted 7 lines
sizeof(req), sizeof(resp));
Quoted 6 lines
drivers/infiniband/hw/mana/main.c +6
-0+ req.hdr.req.msg_version =GDMA_MESSAGE_V2 ; + req.hdr.dev_id = gc->mana_ib.dev_id; + req.notify_eq_id = mdev->fatal_err_eq->id; + + err = mana_gd_send_request(gc, sizeof(req), &req, + sizeof(resp),&resp);
Quoted 15 lines
Quoted 15 lines
drivers/infiniband/hw/mana/main.c static void · line 609
Quoted 3 lines
drivers/infiniband/hw/mana/main.c +3
-0+ if (err) { + ibdev_err(&mdev->ib_dev, "Failed to create + RNIC adapter err %d",err);
Quoted 11 lines
drivers/infiniband/hw/mana/main.c +4
-0+ goto cleanup; + } + mdev->adapter_handle = resp.adapter; +return ; cleanup:@@ -599,5 +616,19 @@ voidmana_ib_gd_create_rnic_adapter(struct
mana_ib_dev*mdev)Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
drivers/infiniband/hw/mana/main.c
Quoted 11 lines
void mana_ib_gd_destroy_rnic_adapter(struct
mana_ib_dev
*mdev) {drivers/infiniband/hw/mana/main.c +9
-0+struct mana_rnic_destroy_adapter_resp resp = {}; +struct mana_rnic_destroy_adapter_req req = {}; +struct gdma_context *gc; + +if (!rnic_is_enabled (mdev)) +return ; + + gc =mdev_to_gc (mdev); + mana_gd_init_req_hdr(&req.hdr,
MANA_IB_DESTROY_ADAPTER,Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
sizeof(req), sizeof(resp));
Quoted 5 lines
drivers/infiniband/hw/mana/main.c +5
-0+ req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + + mana_gd_send_request(gc, sizeof(req), &req, + sizeof(resp),&resp);
Quoted 56 lines
Quoted 56 lines
Quoted 8 lines
Quoted 8 lines
drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_adapter_caps { struct
Quoted 8 lines
+ mdev->adapter_handle = INVALID_MANA_HANDLE; mana_ib_destroy_eqs(mdev); } diff --git
a/drivers/infiniband/hw/mana/mana_ib.hb/drivers/infiniband/hw/mana/mana_ib.h
index a4b94ee..96454cf 100644drivers/infiniband/hw/mana/mana_ib.h +0
-0@@ -48,6 +48,7 @@ struct mana_ib_adapter_caps { struct
mana_ib_dev{Quoted 47 lines
Quoted 47 lines
Quoted 47 lines
drivers/infiniband/hw/mana/mana_ib.h +2
-1struct ib_device ib_dev;struct gdma_dev *gdma_dev; +mana_handle_t adapter_handle;struct gdma_queue *fatal_err_eq; struct mana_ib_adapter_caps adapter_caps; }; @@ -115 ,6 +116,8 @@ struct mana_ib_rwq_ind_table {enum mana_ib_command_code { MANA_IB_GET_ADAPTER_CAP = 0x30001,drivers/infiniband/hw/mana/mana_ib.h +2
-0+MANA_IB_CREATE_ADAPTER =0x30002 , + MANA_IB_DESTROY_ADAPTER = 0x30003, }; struct mana_ib_query_adapter_caps_req { @@ -143,6 +146,32@@ struct mana_ib_query_adapter_caps_resp { u32 max_inline_data_size; }; /* HW Data */drivers/infiniband/hw/mana/mana_ib.h +22
-0+struct mana_rnic_create_adapter_req { +struct gdma_req_hdr hdr; +u32 notify_eq_id; +u32 reserved; +u64 feature_flags; +};/*HW Data */ + +struct mana_rnic_create_adapter_resp { +struct gdma_resp_hdr hdr; +mana_handle_t adapter; };/* HW Data */ + +struct mana_rnic_destroy_adapter_req { +struct gdma_req_hdr hdr; +mana_handle_t adapter; };/*HW Data */ + +struct mana_rnic_destroy_adapter_resp { +struct gdma_resp_hdr hdr; };/* HW Data */ + +static inline bool rnic_is_enabled (struct mana_ib_dev *mdev) { +return mdev->adapter_handle !=INVALID_MANA_HANDLE ; +} + static inline struct gdma_context *mdev_to_gc(struct
mana_ib_dev*mdev) { return mdev->gdma_dev->gdma_context;--
1.8.3.1
Re: [PATCH rdma-next v2 2/5] RDMA/mana_ib: Create and destroy rnic adapter
Quoted 7 lines · Konstantin Taranov
Quoted 3 lines
Unless you have a good explanation why you can add new FW command to configure RNIC, but can't add FW command to query if RNIC is supported. I'm not keen on adopting this approach.
The main reason was backward compatibility with old firmware that had the aforementioned limitation. Anyway, we will try to internally retire the old firmware and will send the v3 patches without the "try and fail" approach (in 2-3 weeks).
I think this is the right thing to do for these cloud devices that can reliably retire old software. It is how Amazon has been running EFA. Get your deployment in good shape and then get patches comitted upstream. No reason to suffer with backwards compatability forever in the software to save a few weeks.
Jason
PATCH v2 3/5 RDMA/mana_ib: Implement port parameters
In reply to Konstantin TaranovImplement port parameters for RNIC. Only port 1 is RoCEv2 capable.
--- drivers/infiniband/hw/mana/device.c | 2 ++ drivers/infiniband/hw/mana/main.c | 37 +++++++++++++++++++++++++++++++++++- drivers/infiniband/hw/mana/mana_ib.h | 4 ++++ 3 files changed, 42 insertions(+), 1 deletion(-)
drivers/infiniband/hw/mana/device.c +2 -0
@@ -29,12 +29,14 @@ .destroy_rwq_ind_table = mana_ib_destroy_rwq_ind_table, .destroy_wq = mana_ib_destroy_wq, .disassociate_ucontext = mana_ib_disassociate_ucontext, + .get_link_layer = mana_ib_get_link_layer, .get_port_immutable = mana_ib_get_port_immutable, .mmap = mana_ib_mmap, .modify_qp = mana_ib_modify_qp, .modify_wq = mana_ib_modify_wq, .query_device = mana_ib_query_device, .query_gid = mana_ib_query_gid, + .query_pkey = mana_ib_query_pkey, .query_port = mana_ib_query_port, .reg_user_mr = mana_ib_reg_user_mr,
drivers/infiniband/hw/mana/main.c +36 -1
@@ -492,7 +492,42 @@ int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props,int mana_ib_query_port (struct ib_device *ibdev,u32 port,struct ib_port_attr *props) { -/* This version doesn't return port properties */ +struct net_device *ndev =mana_ib_get_netdev (ibdev, port); + +if (!ndev) +return -EINVAL ; + +memset (props,0 ,sizeof (*props)); + props->max_mtu =IB_MTU_4096 ; + props->active_mtu =ib_mtu_int_to_enum (ndev->mtu); + +if (netif_carrier_ok (ndev) &&netif_running (ndev)) { + props->state =IB_PORT_ACTIVE ; + props->phys_state =IB_PORT_PHYS_STATE_LINK_UP ; + }else { + props->state =IB_PORT_DOWN ; + props->phys_state =IB_PORT_PHYS_STATE_DISABLED ; + } + + props->active_width =IB_WIDTH_4X ; + props->active_speed =IB_SPEED_EDR ; + props->pkey_tbl_len =1 ; +if (port ==1 ) + props->gid_tbl_len =16 ; + +return 0 ; +} + +enum rdma_link_layer mana_ib_get_link_layer (struct ib_device *device,u32 port_num) +{ +return IB_LINK_LAYER_ETHERNET ; +} + +int mana_ib_query_pkey (struct ib_device *ibdev,u32 port,u16 index,u16 *pkey) +{ +if (index !=0 ) +return -EINVAL ; + *pkey =IB_DEFAULT_PKEY_FULL ;return 0 ; }
drivers/infiniband/hw/mana/mana_ib.h +4 -0
@@ -262,4 +262,8 @@ int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,void mana_ib_gd_create_rnic_adapter (struct mana_ib_dev *mdev);void mana_ib_gd_destroy_rnic_adapter (struct mana_ib_dev *mdev); + +int mana_ib_query_pkey (struct ib_device *ibdev,u32 port,u16 index,u16 *pkey); + +enum rdma_link_layer mana_ib_get_link_layer (struct ib_device *device,u32 port_num);#endif
Signature · 1 line
1.8.3.1
PATCH v2 4/5 RDMA/mana_ib: Enable RoCE on port 1
In reply to Konstantin TaranovSet netdev and RoCEv2 flag to be used in GID population. We need GIDs of the master netdev and mc->ports stores slave devices.
--- drivers/infiniband/hw/mana/device.c | 15 +++++++++++++++ drivers/infiniband/hw/mana/main.c | 16 ++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-)
drivers/infiniband/hw/mana/device.c +15 -0
@@ -53,6 +53,7 @@ static int mana_ib_probe(struct auxiliary_device *adev, { struct mana_adev *madev = container_of(adev, struct mana_adev, adev);struct gdma_dev *mdev = madev->mdev; +struct net_device *upper_ndev;struct mana_context *mc;struct mana_ib_dev *dev;int ret;
@@ -79,6 +80,20 @@ static int mana_ib_probe(struct auxiliary_device *adev, dev->ib_dev.num_comp_vectors =1 ; dev->ib_dev.dev.parent = mdev->gdma_context->dev; +rcu_read_lock ();/* required to get upper dev */ + upper_ndev =netdev_master_upper_dev_get_rcu (mc->ports[0 ]); +if (!upper_ndev) { +rcu_read_unlock (); +ibdev_err (&dev->ib_dev,"Failed to get master netdev" ); + goto free_ib_device; + } + ret =ib_device_set_netdev (&dev->ib_dev, upper_ndev,1 ); +rcu_read_unlock (); +if (ret) { +ibdev_err (&dev->ib_dev,"Failed to set ib netdev, ret %d" , ret); + goto free_ib_device; + } + ret =mana_gd_register_device (&mdev->gdma_context->mana_ib);if (ret) { ibdev_err(&dev->ib_dev, "Failed to register device, ret %d",
drivers/infiniband/hw/mana/main.c +12 -4
@@ -462,11 +462,19 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)int mana_ib_get_port_immutable (struct ib_device *ibdev,u32 port_num,struct ib_port_immutable *immutable) { -/* -* This version only support RAW_PACKET -* other values need to be filled for other types -*/ + struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); +struct ib_port_attr attr; +int err; + + err =ib_query_port (ibdev, port_num, &attr); +if (err) +return err; + + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; immutable->core_cap_flags =RDMA_CORE_PORT_RAW_PACKET ; +if (port_num ==1 &&rnic_is_enabled (mdev)) + immutable->core_cap_flags |=RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP ;return 0 ; }
Signature · 1 line
1.8.3.1
PATCH v2 5/5 RDMA/mana_ib: Adding and deleting GIDs
In reply to Konstantin TaranovImplement add_gid and del_gid for RNIC.
We support ipv4 and ipv6 addresses.
--- drivers/infiniband/hw/mana/device.c | 2 ++ drivers/infiniband/hw/mana/main.c | 66 ++++++++++++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 37 ++++++++++++++++++++ 3 files changed, 105 insertions(+)
drivers/infiniband/hw/mana/device.c +2 -0
@@ -15,6 +15,7 @@ .driver_id = RDMA_DRIVER_MANA, .uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION, + .add_gid = mana_ib_gd_add_gid, .alloc_pd = mana_ib_alloc_pd, .alloc_ucontext = mana_ib_alloc_ucontext, .create_cq = mana_ib_create_cq,
@@ -23,6 +24,7 @@ .create_wq = mana_ib_create_wq, .dealloc_pd = mana_ib_dealloc_pd, .dealloc_ucontext = mana_ib_dealloc_ucontext, + .del_gid = mana_ib_gd_del_gid, .dereg_mr = mana_ib_dereg_mr, .destroy_cq = mana_ib_destroy_cq, .destroy_qp = mana_ib_destroy_qp,
drivers/infiniband/hw/mana/main.c +66 -0
69 lines not on this page · raw message
drivers/infiniband/hw/mana/mana_ib.h +37 -0
@@ -118,6 +118,7 @@ enum mana_ib_command_code {MANA_IB_GET_ADAPTER_CAP =0x30001 ,MANA_IB_CREATE_ADAPTER =0x30002 ,MANA_IB_DESTROY_ADAPTER =0x30003 , + MANA_IB_CONFIG_IP_ADDR = 0x30004, }; struct mana_ib_query_adapter_caps_req {
@@ -167,6 +168,30 @@ struct mana_rnic_destroy_adapter_resp {struct gdma_resp_hdr hdr; };/* HW Data */ +enum mana_ib_addr_op { +ADDR_OP_ADD =1 , +ADDR_OP_REMOVE , +}; + +enum sgid_entry_type { +SGID_TYPE_INVALID =0 , +SGID_TYPE_IPV4 =1 , +SGID_TYPE_IPV6 =2 , +SGID_TYPE_HYBRID =3 +}; + +struct mana_rnic_config_addr_req { +struct gdma_req_hdr hdr; +mana_handle_t adapter; +enum mana_ib_addr_op op; +enum sgid_entry_type sgid_type; +u8 ip_addr[16 ]; +};/* HW Data */ + +struct mana_rnic_config_addr_resp { +struct gdma_resp_hdr hdr; +};/* HW Data */ +static inline bool rnic_is_enabled (struct mana_ib_dev *mdev) {return mdev->adapter_handle !=INVALID_MANA_HANDLE ;
@@ -188,6 +213,14 @@ static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32return mc->ports[port -1 ]; } +static inline void copy_in_reverse (u8 *dst,const u8 *src,u32 size) +{ +u32 i; + +for (i =0 ; i < size; i++) + dst[size -1 - i] = src[i]; +} +int mana_ib_install_cq_cb (struct mana_ib_dev *mdev,struct mana_ib_cq *cq); int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
@@ -266,4 +299,8 @@ int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,int mana_ib_query_pkey (struct ib_device *ibdev,u32 port,u16 index,u16 *pkey);enum rdma_link_layer mana_ib_get_link_layer (struct ib_device *device,u32 port_num); + +int mana_ib_gd_add_gid (const struct ib_gid_attr *attr,void **context); + +int mana_ib_gd_del_gid (const struct ib_gid_attr *attr,void **context);#endif
Signature · 1 line
1.8.3.1
Re: [PATCH rdma-next v2 5/5] RDMA/mana_ib: Adding and deleting GIDs
Quoted 40 lines · Konstantin Taranov
Implement
add_gidanddel_gidfor RNIC.
We support ipv4 and ipv6 addresses.Signed-off-by Konstantin Taranov <[email protected]>--- drivers/infiniband/hw/mana/device.c | 2 ++ drivers/infiniband/hw/mana/main.c | 66 ++++++++++++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 37 ++++++++++++++++++++ 3 files changed, 105 insertions(+)drivers/infiniband/hw/mana/device.c +2
-0@@ -15,6 +15,7 @@ .driver_id = RDMA_DRIVER_MANA, .uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION, + .add_gid = mana_ib_gd_add_gid, .alloc_pd = mana_ib_alloc_pd, .alloc_ucontext = mana_ib_alloc_ucontext, .create_cq = mana_ib_create_cq,@@ -23,6 +24,7 @@ .create_wq = mana_ib_create_wq, .dealloc_pd = mana_ib_dealloc_pd, .dealloc_ucontext = mana_ib_dealloc_ucontext, + .del_gid = mana_ib_gd_del_gid, .dereg_mr = mana_ib_dereg_mr, .destroy_cq = mana_ib_destroy_cq, .destroy_qp = mana_ib_destroy_qp,drivers/infiniband/hw/mana/main.c +2
-0@@ -675,3 +675,69 @@ void mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev) mdev->adapter_handle =INVALID_MANA_HANDLE ;mana_ib_destroy_eqs (mdev); } + +int mana_ib_gd_add_gid (const struct ib_gid_attr *attr,void **context)
drivers/infiniband/hw/mana/main.c void mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev) · line 689
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +10
-0+{ + struct mana_ib_dev *mdev = container_of(attr->device, struct mana_ib_dev, ib_dev); +enum rdma_network_type ntype =rdma_gid_attr_network_type (attr); +struct mana_rnic_config_addr_resp resp = {}; +struct gdma_context *gc =mdev_to_gc (mdev); +struct mana_rnic_config_addr_req req = {}; +int err; + +if (!rnic_is_enabled (mdev)) +return -EINVAL ;
Set .add_gid./del_gid callbacks only when RNIC is enabled. ib_set_device_ops() allows partial set of the callbacks.
Quoted 70 lines
drivers/infiniband/hw/mana/main.c +54
-0+ +if (ntype !=RDMA_NETWORK_IPV4 && ntype !=RDMA_NETWORK_IPV6 ) { +ibdev_dbg (&mdev->ib_dev,"Unsupported rdma network type %d" , ntype); +return -EINVAL ; + } + +mana_gd_init_req_hdr (&req.hdr,MANA_IB_CONFIG_IP_ADDR ,sizeof (req),sizeof (resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + req.op =ADDR_OP_ADD ; + req.sgid_type = (ntype ==RDMA_NETWORK_IPV6 ) ?SGID_TYPE_IPV6 :SGID_TYPE_IPV4 ; +copy_in_reverse (req.ip_addr, attr->gid.raw,sizeof (union ib_gid )); + + err =mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); +if (err) { +ibdev_err (&mdev->ib_dev,"Failed to config IP addr err %d\n" , err); +return err; + } + +return 0 ; +} + +int mana_ib_gd_del_gid (const struct ib_gid_attr *attr,void **context) +{ + struct mana_ib_dev *mdev = container_of(attr->device, struct mana_ib_dev, ib_dev); +enum rdma_network_type ntype =rdma_gid_attr_network_type (attr); +struct mana_rnic_config_addr_resp resp = {}; +struct gdma_context *gc =mdev_to_gc (mdev); +struct mana_rnic_config_addr_req req = {}; +int err; + +if (!rnic_is_enabled (mdev)) +return -EINVAL ; + +if (ntype !=RDMA_NETWORK_IPV4 && ntype !=RDMA_NETWORK_IPV6 ) { +ibdev_dbg (&mdev->ib_dev,"Unsupported rdma network type %d" , ntype); +return -EINVAL ; + } + +mana_gd_init_req_hdr (&req.hdr,MANA_IB_CONFIG_IP_ADDR ,sizeof (req),sizeof (resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + req.op =ADDR_OP_REMOVE ; + req.sgid_type = (ntype ==RDMA_NETWORK_IPV6 ) ?SGID_TYPE_IPV6 :SGID_TYPE_IPV4 ; +copy_in_reverse (req.ip_addr, attr->gid.raw,sizeof (union ib_gid )); + + err =mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); +if (err) { +ibdev_err (&mdev->ib_dev,"Failed to config IP addr err %d\n" , err); +return err; + } + +return 0 ; +}drivers/infiniband/hw/mana/mana_ib.h +1
-0@@ -118,6 +118,7 @@ enum mana_ib_command_code {MANA_IB_GET_ADAPTER_CAP =0x30001 ,MANA_IB_CREATE_ADAPTER =0x30002 ,MANA_IB_DESTROY_ADAPTER =0x30003 , + MANA_IB_CONFIG_IP_ADDR = 0x30004, }; struct mana_ib_query_adapter_caps_req {@@ -167,6 +168,30 @@ struct mana_rnic_destroy_adapter_resp {struct gdma_resp_hdr hdr; };/* HW Data */
drivers/infiniband/hw/mana/mana_ib.h struct mana_rnic_destroy_adapter_resp { · line 180
Quoted 10 lines
drivers/infiniband/hw/mana/mana_ib.h +10
-0+enum mana_ib_addr_op { +ADDR_OP_ADD =1 , +ADDR_OP_REMOVE , +}; + +enum sgid_entry_type { +SGID_TYPE_INVALID =0 , +SGID_TYPE_IPV4 =1 , +SGID_TYPE_IPV6 =2 , +SGID_TYPE_HYBRID =3
This is not used, please remove it.
Thanks