Retrieve HW GID context from ib_gid_attr
Subsystems (2)
Selvin Xavier
maintainerv2 cover Retrieve HW GID context from ib_gid_attr
Provide an option for vendor drivers to get the HW GID context from the ib_gid_attr during modify_qp and create_ah. Required for drivers/HW that maintains HW gid index different than the host sgid_index.
Please review.
Thanks,
Selvin Xavier
v1 -> v2:
Addressed review comments from Parav
Selvin Xavier (2):
IB/core: Add option to retrieve driver gid context from ib_gid_attr RDMA/bnxt_re: Retrieve the driver gid context from gid_attr
drivers/infiniband/core/cache.c | 79 ++++++++++++++++++-------------- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 28 +++++------ include/rdma/ib_verbs.h | 5 ++ 3 files changed, 65 insertions(+), 47 deletions(-)
Signature · 1 line
2.5.5
Selvin Xavier
maintainerPATCH v2 1/2 IB/core: Add option to retrieve driver gid context from ib_gid_attr
Provide an option to retrieve the driver gid context from ib_gid_attr structure. Introduce ib_gid_attr_info structure which include both gid_attr and the GID's HW context. Replace the attr and context members of ib_gid_table_entry with the new ib_gid_attr_info structure. Vendor drivers can refer to its own HW gid context using the container_of macro.
--- drivers/infiniband/core/cache.c | 79 +++++++++++++++++++++++------------------ include/rdma/ib_verbs.h | 5 +++ 2 files changed, 50 insertions(+), 34 deletions(-)
drivers/infiniband/core/cache.c +45 -34
@@ -86,8 +86,7 @@ struct roce_gid_ndev_storage {struct ib_gid_table_entry {struct kref kref;struct work_struct del_work; -struct ib_gid_attr attr; -void *context; +struct ib_gid_attr_info attr_info;/* Store the ndev pointer to release reference later on in * call_rcu context because by that time gid_table_entry * and attr might be already freed. So keep a copy of it.
@@ -233,12 +232,13 @@ static void put_gid_ndev(struct rcu_head *head)static void free_gid_entry_locked (struct ib_gid_table_entry *entry) { -struct ib_device *device = entry->attr.device; -u8 port_num = entry->attr.port_num; +struct ib_device *device = entry->attr_info.attr.device; +u8 port_num = entry->attr_info.attr.port_num;struct ib_gid_table *table =rdma_gid_table (device, port_num);dev_dbg (&device->dev,"%s port=%d index=%d gid %pI6\n" , __func__, - port_num, entry->attr.index, entry->attr.gid.raw); + port_num, entry->attr_info.attr.index, + entry->attr_info.attr.gid.raw);write_lock_irq (&table->rwlock);
@@ -248,8 +248,8 @@ static void free_gid_entry_locked(struct ib_gid_table_entry *entry)* If new entry in table is added by the time we free here, * don't overwrite the table entry. */ -if (entry == table->data_vec[entry->attr.index]) - table->data_vec[entry->attr.index] =NULL ; +if (entry == table->data_vec[entry->attr_info.attr.index]) + table->data_vec[entry->attr_info.attr.index] =NULL ;/* Now this index is ready to be allocated */ write_unlock_irq (&table->rwlock);
@@ -278,8 +278,8 @@ static void free_gid_work(struct work_struct *work) {struct ib_gid_table_entry *entry = container_of(work, struct ib_gid_table_entry, del_work); -struct ib_device *device = entry->attr.device; -u8 port_num = entry->attr.port_num; +struct ib_device *device = entry->attr_info.attr.device; +u8 port_num = entry->attr_info.attr.port_num;struct ib_gid_table *table =rdma_gid_table (device, port_num);mutex_lock (&table->lock);
@@ -309,7 +309,7 @@ alloc_gid_entry(const struct ib_gid_attr *attr) entry->ndev_storage->ndev = ndev; }kref_init (&entry->kref); -memcpy (&entry->attr, attr,sizeof (*attr)); +memcpy (&entry->attr_info.attr, attr,sizeof (*attr));INIT_WORK (&entry->del_work, free_gid_work); entry->state =GID_TABLE_ENTRY_INVALID ;return entry;
@@ -320,13 +320,15 @@ static void store_gid_entry(struct ib_gid_table *table, { entry->state =GID_TABLE_ENTRY_VALID ; -dev_dbg (&entry->attr.device->dev,"%s port=%d index=%d gid %pI6\n" , - __func__, entry->attr.port_num, entry->attr.index, - entry->attr.gid.raw); +dev_dbg (&entry->attr_info.attr.device->dev, +"%s port=%d index=%d gid %pI6\n" , + __func__, entry->attr_info.attr.port_num, + entry->attr_info.attr.index, + entry->attr_info.attr.gid.raw);lockdep_assert_held (&table->lock);write_lock_irq (&table->rwlock); - table->data_vec[entry->attr.index] = entry; + table->data_vec[entry->attr_info.attr.index] = entry;write_unlock_irq (&table->rwlock); }
@@ -347,7 +349,7 @@ static void put_gid_entry_locked(struct ib_gid_table_entry *entry)static int add_roce_gid (struct ib_gid_table_entry *entry) { -const struct ib_gid_attr *attr = &entry->attr; +const struct ib_gid_attr *attr = &entry->attr_info.attr;int ret;if (!attr->ndev) {
@@ -356,7 +358,8 @@ static int add_roce_gid(struct ib_gid_table_entry *entry)return -EINVAL ; }if (rdma_cap_roce_gid_table (attr->device, attr->port_num)) { - ret = attr->device->ops.add_gid (attr, &entry->context); + ret = attr->device->ops.add_gid (attr, + &entry->attr_info.context);if (ret) { dev_err(&attr->device->dev, "%s GID add failed port=%d index=%d\n",
@@ -385,7 +388,7 @@ static void del_gid(struct ib_device *ib_dev, u8 port,lockdep_assert_held (&table->lock);dev_dbg (&ib_dev->dev,"%s port=%d index=%d gid %pI6\n" , __func__, port, - ix, table->data_vec[ix]->attr.gid.raw); + ix, table->data_vec[ix]->attr_info.attr.gid.raw);write_lock_irq (&table->rwlock); entry = table->data_vec[ix];
@@ -400,12 +403,13 @@ static void del_gid(struct ib_device *ib_dev, u8 port, ndev_storage = entry->ndev_storage;if (ndev_storage) { entry->ndev_storage =NULL ; -rcu_assign_pointer (entry->attr.ndev,NULL ); +rcu_assign_pointer (entry->attr_info.attr.ndev,NULL );call_rcu (&ndev_storage->rcu_head, put_gid_ndev); }if (rdma_cap_roce_gid_table (ib_dev, port)) - ib_dev->ops.del_gid (&entry->attr, &entry->context); + ib_dev->ops.del_gid (&entry->attr_info.attr, + &entry->attr_info.context);put_gid_entry_locked (entry); }
@@ -508,13 +512,13 @@ static int find_gid(struct ib_gid_table *table, const union ib_gid *gid,if (found >=0 )continue ; - attr = &data->attr; + attr = &data->attr_info.attr;if (mask &GID_ATTR_FIND_MASK_GID_TYPE && attr->gid_type != val->gid_type)continue ;if (mask &GID_ATTR_FIND_MASK_GID && -memcmp (gid, &data->attr.gid,sizeof (*gid))) +memcmp (gid, &data->attr_info.attr.gid,sizeof (*gid)))continue ; if (mask & GID_ATTR_FIND_MASK_NETDEV &&
@@ -648,7 +652,7 @@ int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u8 port,for (ix =0 ; ix < table->sz; ix++) {if (is_gid_entry_valid (table->data_vec[ix]) && - table->data_vec[ix]->attr.ndev == ndev) { + table->data_vec[ix]->attr_info.attr.ndev == ndev) {del_gid (ib_dev, port, table, ix); deleted = true; }
@@ -703,7 +707,7 @@ rdma_find_gid_by_port(struct ib_device *ib_dev, local_index =find_gid (table, gid, &val, false, mask,NULL );if (local_index >=0 ) {get_gid_entry (table->data_vec[local_index]); - attr = &table->data_vec[local_index]->attr; + attr = &table->data_vec[local_index]->attr_info.attr;read_unlock_irqrestore (&table->rwlock, flags);return attr; }
@@ -753,12 +757,12 @@ const struct ib_gid_attr *rdma_find_gid_by_filter(if (!is_gid_entry_valid (entry))continue ; -if (memcmp (gid, &entry->attr.gid,sizeof (*gid))) +if (memcmp (gid, &entry->attr_info.attr.gid,sizeof (*gid)))continue ; -if (filter (gid, &entry->attr, context)) { +if (filter (gid, &entry->attr_info.attr, context)) {get_gid_entry (entry); - res = &entry->attr; + res = &entry->attr_info.attr;break ; } }
@@ -964,7 +968,7 @@ int rdma_query_gid(struct ib_device *device, u8 port_num, !is_gid_entry_valid(table->data_vec[index])) goto done; -memcpy (gid, &table->data_vec[index]->attr.gid,sizeof (*gid)); +memcpy (gid, &table->data_vec[index]->attr_info.attr.gid,sizeof (*gid)); res =0 ; done:
@@ -1011,7 +1015,7 @@ const struct ib_gid_attr *rdma_find_gid(struct ib_device *device,const struct ib_gid_attr *attr;get_gid_entry (table->data_vec[index]); - attr = &table->data_vec[index]->attr; + attr = &table->data_vec[index]->attr_info.attr;read_unlock_irqrestore (&table->rwlock, flags);return attr; }
@@ -1210,7 +1214,7 @@ rdma_get_gid_attr(struct ib_device *device, u8 port_num, int index) goto done;get_gid_entry (table->data_vec[index]); - attr = &table->data_vec[index]->attr; + attr = &table->data_vec[index]->attr_info.attr; done:read_unlock_irqrestore (&table->rwlock, flags);return attr;
@@ -1230,8 +1234,10 @@ EXPORT_SYMBOL(rdma_get_gid_attr);*/ void rdma_put_gid_attr (const struct ib_gid_attr *attr) { +struct ib_gid_attr_info *info = + container_of(attr, struct ib_gid_attr_info, attr);struct ib_gid_table_entry *entry = - container_of(attr, struct ib_gid_table_entry, attr); + container_of(info, struct ib_gid_table_entry, attr_info);put_gid_entry (entry); }
@@ -1249,8 +1255,10 @@ EXPORT_SYMBOL(rdma_put_gid_attr);*/ void rdma_hold_gid_attr (const struct ib_gid_attr *attr) { +struct ib_gid_attr_info *info = + container_of(attr, struct ib_gid_attr_info, attr);struct ib_gid_table_entry *entry = - container_of(attr, struct ib_gid_table_entry, attr); + container_of(info, struct ib_gid_table_entry, attr_info);get_gid_entry (entry); }
@@ -1270,11 +1278,14 @@ EXPORT_SYMBOL(rdma_hold_gid_attr);*/ struct net_device *rdma_read_gid_attr_ndev_rcu (const struct ib_gid_attr *attr) { +struct ib_gid_attr_info *info = + container_of(attr, struct ib_gid_attr_info, attr);struct ib_gid_table_entry *entry = - container_of(attr, struct ib_gid_table_entry, attr); -struct ib_device *device = entry->attr.device; + container_of(info, struct ib_gid_table_entry, + attr_info); +struct ib_device *device = entry->attr_info.attr.device;struct net_device *ndev =ERR_PTR (-ENODEV ); -u8 port_num = entry->attr.port_num; +u8 port_num = entry->attr_info.attr.port_num;struct ib_gid_table *table;unsigned long flags;bool valid;
include/rdma/ib_verbs.h +5 -0
@@ -174,6 +174,11 @@ struct ib_gid_attr {u8 port_num; }; +struct ib_gid_attr_info { +struct ib_gid_attr attr; +void *context; +}; +enum {/* set the local administered indication */ IB_SA_WELL_KNOWN_GUID =BIT_ULL (57 ) |2 ,
Signature · 1 line
2.5.5
Re: [PATCH for-next v2 1/2] IB/core: Add option to retrieve driver gid context from ib_gid_attr
Quoted 6 lines · Selvin Xavier
Provide an option to retrieve the driver gid context from
ib_gid_attrstructure. Introduceib_gid_attr_infostructure which include bothgid_attrand the GID's HW context. Replace the attr and context members ofib_gid_table_entrywith the newib_gid_attr_infostructure. Vendor drivers can refer to its own HW gid context using thecontainer_ofmacro.
This seems really weird. Why are we adding a new struct instead of adding context to the normal gid_attr, or adding some 'get_ib_attr_priv' call?
Jason
Re: [PATCH for-next v2 1/2] IB/core: Add option to retrieve driver gid context from ib_gid_attr
Quoted 11 lines · Jason Gunthorpe
Quoted 6 lines · Selvin Xavier
Provide an option to retrieve the driver gid context from
ib_gid_attrstructure. Introduceib_gid_attr_infostructure which include bothgid_attrand the GID's HW context. Replace the attr and context members ofib_gid_table_entrywith the newib_gid_attr_infostructure. Vendor drivers can refer to its own HW gid context using thecontainer_ofmacro.This seems really weird. Why are we adding a new struct instead of adding context to the normal
gid_attr, or adding some 'get_ib_attr_priv' call?
Rest of the stack didn't need to touch context, so it is added only as vendor driver facing container_of().
Instead I guess a new symbol as rdma_get_gid_attr_context() can be added too.
Selvin Xavier
maintainerRe: [PATCH for-next v2 1/2] IB/core: Add option to retrieve driver gid context from ib_gid_attr
Quoted 10 lines · Parav Pandit
Quoted 8 lines · Jason Gunthorpe
Quoted 6 lines · Selvin Xavier
Provide an option to retrieve the driver gid context from
ib_gid_attrstructure. Introduceib_gid_attr_infostructure which include bothgid_attrand the GID's HW context. Replace the attr and context members ofib_gid_table_entrywith the newib_gid_attr_infostructure. Vendor drivers can refer to its own HW gid context using thecontainer_ofmacro.
Quoted 6 lines
Quoted 3 lines
This seems really weird. Why are we adding a new struct instead of adding context to the normal
gid_attr, or adding some 'get_ib_attr_priv' call?Rest of the stack didn't need to touch context, so it is added only as vendor driver facing
container_of().
Added the new structure since I didn't want to move the private structure ib_gid_table_entry to a header file.
Quoted 3 lines
Instead I guess a new symbol as
rdma_get_gid_attr_context()can be added too.
I am okay with both adding context to gid_attr struct or adding a symbol. Let me know your preference.
Or shall i handle this inside bnxt_re itself. Not sure whether any other drivers intend to use this.
Re: [PATCH for-next v2 1/2] IB/core: Add option to retrieve driver gid context from ib_gid_attr
Quoted 6 lines · Selvin Xavier
Quoted 2 lines
Instead I guess a new symbol as
rdma_get_gid_attr_context()can be added too.I am okay with both adding context to
gid_attrstruct or adding a symbol. Let me know your preference.
Or shall i handle this insidebnxt_reitself. Not sure whether any other drivers intend to use this.
Having a function to return the same void * that is passed to the driver ops functions seems reasonable and small to me.
But you could also spruce this up a bit and have it work more like a true 'priv' and get rid of that void *..
The container_of thing is really odd
Jason
Selvin Xavier
maintainerRe: [PATCH for-next v2 1/2] IB/core: Add option to retrieve driver gid context from ib_gid_attr
Quoted 11 lines · Jason Gunthorpe
Quoted 6 lines · Selvin Xavier
Quoted 2 lines
Instead I guess a new symbol as
rdma_get_gid_attr_context()can be added too.I am okay with both adding context to
gid_attrstruct or adding a symbol. Let me know your preference.
Or shall i handle this insidebnxt_reitself. Not sure whether any other drivers intend to use this.Having a function to return the same void * that is passed to the driver ops functions seems reasonable and small to me.
Posting a patch with a symbol that returns void *
Thanks
Quoted 7 lines
But you could also spruce this up a bit and have it work more like a true 'priv' and get rid of that void *..
The
container_ofthing is really oddJason
Selvin Xavier
maintainerPATCH v2 2/2 RDMA/bnxt_re: Retrieve the driver gid context from gid_attr
In reply to Selvin XavierUse the container_of macro to retrieve the driver gid context.
--- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-)
drivers/infiniband/hw/bnxt_re/ib_verbs.c +15 -13
@@ -640,6 +640,8 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr,struct bnxt_re_dev *rdev = pd->rdev;const struct ib_gid_attr *sgid_attr; struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); +struct ib_gid_attr_info *info; +struct bnxt_re_gid_ctx *ctx;u8 nw_type;int rc;
@@ -651,22 +653,21 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr, ah->rdev = rdev; ah->qplib_ah.pd = &pd->qplib_pd; + sgid_attr = grh->sgid_attr; + + info = container_of(sgid_attr, struct ib_gid_attr_info, attr); + ctx = info->context; +/* Supply the configuration for the HW */ memcpy (ah->qplib_ah.dgid.data, grh->dgid.raw,sizeof (union ib_gid )); -/* -* If RoCE V2 is enabled, stack will have two entries for -* each GID entry. Avoiding this duplicte entry in HW. Dividing -* the GID index by 2 for RoCE V2 -*/ - ah->qplib_ah.sgid_index = grh->sgid_index /2 ; + ah->qplib_ah.sgid_index = ctx->idx; ah->qplib_ah.host_sgid_index = grh->sgid_index; ah->qplib_ah.traffic_class = grh->traffic_class; ah->qplib_ah.flow_label = grh->flow_label; ah->qplib_ah.hop_limit = grh->hop_limit; ah->qplib_ah.sl =rdma_ah_get_sl (ah_attr); - sgid_attr = grh->sgid_attr;/* Get network header type for this GID */ nw_type =rdma_gid_attr_network_type (sgid_attr); ah->qplib_ah.nw_type =bnxt_re_stack_to_dev_nw_type (nw_type);
@@ -1521,6 +1522,8 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,struct bnxt_re_dev *rdev = qp->rdev;struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;enum ib_qp_state curr_qp_state, new_qp_state; +struct ib_gid_attr_info *info; +struct bnxt_re_gid_ctx *ctx;int rc, entries;unsigned int flags;u8 nw_type;
@@ -1592,6 +1595,10 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,rdma_ah_read_grh (&qp_attr->ah_attr);const struct ib_gid_attr *sgid_attr; + sgid_attr = qp_attr->ah_attr.grh.sgid_attr; + info = container_of(sgid_attr, struct ib_gid_attr_info, attr); + ctx = info->context; + qp->qplib_qp.modify_flags |=CMDQ_MODIFY_QP_MODIFY_MASK_DGID |CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
@@ -1602,11 +1609,7 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,memcpy (qp->qplib_qp.ah.dgid.data, grh->dgid.raw,sizeof (qp->qplib_qp.ah.dgid.data)); qp->qplib_qp.ah.flow_label = grh->flow_label; -/* If RoCE V2 is enabled, stack will have two entries for -* each GID entry. Avoiding this duplicte entry in HW. Dividing -* the GID index by 2 for RoCE V2 -*/ - qp->qplib_qp.ah.sgid_index = grh->sgid_index /2 ; + qp->qplib_qp.ah.sgid_index = ctx->idx; qp->qplib_qp.ah.host_sgid_index = grh->sgid_index; qp->qplib_qp.ah.hop_limit = grh->hop_limit; qp->qplib_qp.ah.traffic_class = grh->traffic_class;
@@ -1614,7 +1617,6 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,ether_addr_copy (qp->qplib_qp.ah.dmac, qp_attr->ah_attr.roce.dmac); - sgid_attr = qp_attr->ah_attr.grh.sgid_attr; rc =rdma_read_gid_l2_fields (sgid_attr,NULL , &qp->qplib_qp.smac[0 ]);if (rc)
Signature · 1 line
2.5.5