RDMA/hns: Optimize codes related to multi-hop addressing
Subsystems (2)
cover RDMA/hns: Optimize codes related to multi-hop addressing
Codes about getting/putting HEM(Hardware Entry Memory) in multi-hop addressing are not clear enough, so optimize by spliting and encapsulating them into new functions.
Xi Wang (2):
RDMA/hns: Optimize mhop get flow for multi-hop addressing RDMA/hns: Optimize mhop put flow for multi-hop addressing
drivers/infiniband/hw/hns/hns_roce_hem.c | 458 ++++++++++++++++--------------- 1 file changed, 243 insertions(+), 215 deletions(-)
Signature · 1 line
2.8.1
PATCH 1/2 RDMA/hns: Optimize mhop get flow for multi-hop addressing
From: Xi Wang <[email protected]>
Splits hns_roce_table_mhop_get() into 4 sub-functions to make the code flow clearer.
--- drivers/infiniband/hw/hns/hns_roce_hem.c | 297 +++++++++++++++++++------------ 1 file changed, 182 insertions(+), 115 deletions(-)
drivers/infiniband/hw/hns/hns_roce_hem.c +182 -115
@@ -39,6 +39,16 @@#define DMA_ADDR_T_SHIFT 12#define BT_BA_SHIFT 32 +#define HEM_INDEX_BUF BIT(0) +#define HEM_INDEX_L0 BIT(1) +#define HEM_INDEX_L1 BIT(2) +struct hns_roce_hem_index { +u64 buf; +u64 l0; +u64 l1; +u32 inited;/* indicate which index is available */ +}; +bool hns_roce_check_whether_mhop (struct hns_roce_dev *hr_dev,u32 type) {int hop_num =0 ;
@@ -434,178 +444,235 @@ static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,return ret; } -static int hns_roce_table_mhop_get (struct hns_roce_dev *hr_dev, -struct hns_roce_hem_table *table, -unsigned long obj) +static int calc_hem_config (struct hns_roce_dev *hr_dev, +struct hns_roce_hem_table *table,unsigned long obj, +struct hns_roce_hem_mhop *mhop, +struct hns_roce_hem_index *index) { -struct device *dev = hr_dev->dev; -struct hns_roce_hem_mhop mhop; -struct hns_roce_hem_iter iter; -u32 buf_chunk_size; -u32 bt_chunk_size; +struct ib_device *ibdev = &hr_dev->ib_dev; +unsigned long mhop_obj = obj; +u32 l0_idx, l1_idx, l2_idx;u32 chunk_ba_num; -u32 hop_num; -u32 size;u32 bt_num; -u64 hem_idx; -u64 bt_l1_idx =0 ; -u64 bt_l0_idx =0 ; -u64 bt_ba; -unsigned long mhop_obj = obj; -int bt_l1_allocated =0 ; -int bt_l0_allocated =0 ; -int step_idx;int ret; - ret =hns_roce_calc_hem_mhop (hr_dev, table, &mhop_obj, &mhop); + ret =hns_roce_calc_hem_mhop (hr_dev, table, &mhop_obj, mhop);if (ret)return ret; - buf_chunk_size = mhop.buf_chunk_size; - bt_chunk_size = mhop.bt_chunk_size; - hop_num = mhop.hop_num; - chunk_ba_num = bt_chunk_size /BA_BYTE_LEN ; - - bt_num =hns_roce_get_bt_num (table->type, hop_num); + l0_idx = mhop->l0_idx; + l1_idx = mhop->l1_idx; + l2_idx = mhop->l2_idx; + chunk_ba_num = mhop->bt_chunk_size /BA_BYTE_LEN ; + bt_num =hns_roce_get_bt_num (table->type, mhop->hop_num);switch (bt_num) {case 3 : - hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + - mhop.l1_idx * chunk_ba_num + mhop.l2_idx; - bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; - bt_l0_idx = mhop.l0_idx; + index->l1 = l0_idx * chunk_ba_num + l1_idx; + index->l0 = l0_idx; + index->buf = l0_idx * chunk_ba_num * chunk_ba_num + + l1_idx * chunk_ba_num + l2_idx;break ;case 2 : - hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; - bt_l0_idx = mhop.l0_idx; + index->l0 = l0_idx; + index->buf = l0_idx * chunk_ba_num + l1_idx;break ;case 1 : - hem_idx = mhop.l0_idx; + index->buf = l0_idx;break ;default : -dev_err (dev,"Table %d not support hop_num = %d!\n" , - table->type, hop_num); +ibdev_err (ibdev,"Table %d not support mhop.hop_num = %d!\n" , + table->type, mhop->hop_num);return -EINVAL ; } -if (unlikely (hem_idx >= table->num_hem)) { -dev_err (dev,"Table %d exceed hem limt idx = %llu,max = %lu!\n" , - table->type, hem_idx, table->num_hem); +if (unlikely (index->buf >= table->num_hem)) { +ibdev_err (ibdev,"Table %d exceed hem limt idx %llu,max %lu!\n" , + table->type, index->buf, table->num_hem);return -EINVAL ; } -mutex_lock (&table->mutex); +return 0 ; +} -if (table->hem[hem_idx]) { - ++table->hem[hem_idx]->refcount; - goto out; +static void free_mhop_hem (struct hns_roce_dev *hr_dev, +struct hns_roce_hem_table *table, +struct hns_roce_hem_mhop *mhop, +struct hns_roce_hem_index *index) +{ +u32 bt_size = mhop->bt_chunk_size; +struct device *dev = hr_dev->dev; + +if (index->inited &HEM_INDEX_BUF ) { +hns_roce_free_hem (hr_dev, table->hem[index->buf]); + table->hem[index->buf] =NULL ; + } + +if (index->inited &HEM_INDEX_L1 ) { +dma_free_coherent (dev, bt_size, table->bt_l1[index->l1], + table->bt_l1_dma_addr[index->l1]); + table->bt_l1[index->l1] =NULL ; } +if (index->inited &HEM_INDEX_L0 ) { +dma_free_coherent (dev, bt_size, table->bt_l0[index->l0], + table->bt_l0_dma_addr[index->l0]); + table->bt_l0[index->l0] =NULL ; + } +} + +static int alloc_mhop_hem (struct hns_roce_dev *hr_dev, +struct hns_roce_hem_table *table, +struct hns_roce_hem_mhop *mhop, +struct hns_roce_hem_index *index) +{ +u32 bt_size = mhop->bt_chunk_size; +struct device *dev = hr_dev->dev; +struct hns_roce_hem_iter iter; +gfp_t flag; +u64 bt_ba; +u32 size; +int ret; +/* alloc L1 BA's chunk */ -if ((check_whether_bt_num_3 (table->type, hop_num) || -check_whether_bt_num_2 (table->type, hop_num)) && - !table->bt_l0[bt_l0_idx]) { - table->bt_l0[bt_l0_idx] =dma_alloc_coherent (dev, bt_chunk_size, - &(table->bt_l0_dma_addr[bt_l0_idx]), +if ((check_whether_bt_num_3 (table->type, mhop->hop_num) || +check_whether_bt_num_2 (table->type, mhop->hop_num)) && + !table->bt_l0[index->l0]) { + table->bt_l0[index->l0] =dma_alloc_coherent (dev, bt_size, + &table->bt_l0_dma_addr[index->l0],GFP_KERNEL ); -if (!table->bt_l0[bt_l0_idx]) { +if (!table->bt_l0[index->l0]) { ret = -ENOMEM ; goto out; } - bt_l0_allocated =1 ; - -/* set base address to hardware */ -if (table->type <HEM_TYPE_MTT ) { - step_idx =0 ; -if (hr_dev->hw->set_hem (hr_dev, table, obj, step_idx)) { - ret = -ENODEV ; -dev_err (dev,"set HEM base address to HW failed!\n" ); - goto err_dma_alloc_l1; - } - } + index->inited |=HEM_INDEX_L0 ; }/* alloc L2 BA's chunk */ -if (check_whether_bt_num_3 (table->type, hop_num) && - !table->bt_l1[bt_l1_idx]) { - table->bt_l1[bt_l1_idx] =dma_alloc_coherent (dev, bt_chunk_size, - &(table->bt_l1_dma_addr[bt_l1_idx]), +if (check_whether_bt_num_3 (table->type, mhop->hop_num) && + !table->bt_l1[index->l1]) { + table->bt_l1[index->l1] =dma_alloc_coherent (dev, bt_size, + &table->bt_l1_dma_addr[index->l1],GFP_KERNEL ); -if (!table->bt_l1[bt_l1_idx]) { +if (!table->bt_l1[index->l1]) { ret = -ENOMEM ; - goto err_dma_alloc_l1; - } - bt_l1_allocated =1 ; - *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = - table->bt_l1_dma_addr[bt_l1_idx]; - -/* set base address to hardware */ - step_idx =1 ; -if (hr_dev->hw->set_hem (hr_dev, table, obj, step_idx)) { - ret = -ENODEV ; -dev_err (dev,"set HEM base address to HW failed!\n" ); - goto err_alloc_hem_buf; + goto err_alloc_hem; } + index->inited |=HEM_INDEX_L1 ; + *(table->bt_l0[index->l0] + mhop->l1_idx) = + table->bt_l1_dma_addr[index->l1]; }/* * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC. * alloc bt space chunk for MTT/CQE. */ - size = table->type <HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size; - table->hem[hem_idx] =hns_roce_alloc_hem (hr_dev, - size >>PAGE_SHIFT , - size, - (table->lowmem ?GFP_KERNEL : -GFP_HIGHUSER ) | __GFP_NOWARN); -if (!table->hem[hem_idx]) { + size = table->type <HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size; + flag = (table->lowmem ?GFP_KERNEL :GFP_HIGHUSER ) | __GFP_NOWARN; + table->hem[index->buf] =hns_roce_alloc_hem (hr_dev, size >>PAGE_SHIFT , + size, flag); +if (!table->hem[index->buf]) { ret = -ENOMEM ; - goto err_alloc_hem_buf; + goto err_alloc_hem; } -hns_roce_hem_first (table->hem[hem_idx], &iter); + index->inited |=HEM_INDEX_BUF ; +hns_roce_hem_first (table->hem[index->buf], &iter); bt_ba =hns_roce_hem_addr (&iter); -if (table->type <HEM_TYPE_MTT ) { -if (hop_num ==2 ) { - *(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba; - step_idx =2 ; - }else if (hop_num ==1 ) { - *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba; - step_idx =1 ; - }else if (hop_num ==HNS_ROCE_HOP_NUM_0 ) { - step_idx =0 ; - }else { - ret = -EINVAL ; - goto err_dma_alloc_l1; +if (mhop->hop_num ==2 ) + *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba; +else if (mhop->hop_num ==1 ) + *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; + }else if (mhop->hop_num ==2 ) { + *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; + } + +return 0 ; +err_alloc_hem: +free_mhop_hem (hr_dev, table, mhop, index); +out: +return ret; +} + +static int set_mhop_hem (struct hns_roce_dev *hr_dev, +struct hns_roce_hem_table *table,unsigned long obj, +struct hns_roce_hem_mhop *mhop, +struct hns_roce_hem_index *index) +{ +struct ib_device *ibdev = &hr_dev->ib_dev; +int step_idx; +int ret; + +if (index->inited &HEM_INDEX_L0 ) { + ret = hr_dev->hw->set_hem (hr_dev, table, obj,0 ); +if (ret) { +ibdev_err (ibdev,"set HEM step 0 failed!\n" ); + goto out; } + } -/* set HEM base address to hardware */ -if (hr_dev->hw->set_hem (hr_dev, table, obj, step_idx)) { - ret = -ENODEV ; -dev_err (dev,"set HEM base address to HW failed!\n" ); - goto err_alloc_hem_buf; +if (index->inited &HEM_INDEX_L1 ) { + ret = hr_dev->hw->set_hem (hr_dev, table, obj,1 ); +if (ret) { +ibdev_err (ibdev,"set HEM step 1 failed!\n" ); + goto out; } - }else if (hop_num ==2 ) { - *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba; } - ++table->hem[hem_idx]->refcount; - goto out; +if (index->inited &HEM_INDEX_BUF ) { +if (mhop->hop_num ==HNS_ROCE_HOP_NUM_0 ) + step_idx =0 ; +else + step_idx = mhop->hop_num; + ret = hr_dev->hw->set_hem (hr_dev, table, obj, step_idx); +if (ret) +ibdev_err (ibdev,"set HEM step last failed!\n" ); + } +out: +return ret; +} -err_alloc_hem_buf: -if (bt_l1_allocated) { -dma_free_coherent (dev, bt_chunk_size, table->bt_l1[bt_l1_idx], - table->bt_l1_dma_addr[bt_l1_idx]); - table->bt_l1[bt_l1_idx] =NULL ; +static int hns_roce_table_mhop_get (struct hns_roce_dev *hr_dev, +struct hns_roce_hem_table *table, +unsigned long obj) +{ +struct ib_device *ibdev = &hr_dev->ib_dev; +struct hns_roce_hem_index index = {}; +struct hns_roce_hem_mhop mhop = {}; +int ret; + + ret =calc_hem_config (hr_dev, table, obj, &mhop, &index); +if (ret) { +ibdev_err (ibdev,"calc hem config failed!\n" ); +return ret; } -err_dma_alloc_l1: -if (bt_l0_allocated) { -dma_free_coherent (dev, bt_chunk_size, table->bt_l0[bt_l0_idx], - table->bt_l0_dma_addr[bt_l0_idx]); - table->bt_l0[bt_l0_idx] =NULL ; +mutex_lock (&table->mutex); +if (table->hem[index.buf]) { + ++table->hem[index.buf]->refcount; + goto out; + } + + ret =alloc_mhop_hem (hr_dev, table, &mhop, &index); +if (ret) { +ibdev_err (ibdev,"alloc mhop hem failed!\n" ); + goto out; + } + +/* set HEM base address to hardware */ +if (table->type <HEM_TYPE_MTT ) { + ret =set_mhop_hem (hr_dev, table, obj, &mhop, &index); +if (ret) { +ibdev_err (ibdev,"set HEM address to HW failed!\n" ); + goto err_alloc; + } } + ++table->hem[index.buf]->refcount; + goto out; + +err_alloc: +free_mhop_hem (hr_dev, table, &mhop, &index); out:mutex_unlock (&table->mutex);return ret;
Signature · 1 line
2.8.1
PATCH 2/2 RDMA/hns: Optimize mhop put flow for multi-hop addressing
In reply to Weihang LiFrom: Xi Wang <[email protected]>
Optimizes hns_roce_table_mhop_get() by encapsulating code about clearing hem into clear_mhop_hem(), which will make the code flow clearer.
--- drivers/infiniband/hw/hns/hns_roce_hem.c | 161 ++++++++++++------------------- 1 file changed, 61 insertions(+), 100 deletions(-)
drivers/infiniband/hw/hns/hns_roce_hem.c +61 -100
@@ -94,25 +94,27 @@ bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)return hop_num ? true : false; } -static bool hns_roce_check_hem_null (struct hns_roce_hem **hem,u64 start_idx, -u32 bt_chunk_num,u64 hem_max_num) +static bool hns_roce_check_hem_null (struct hns_roce_hem **hem,u64 hem_idx, +u32 bt_chunk_num,u64 hem_max_num) { +u64 start_idx =round_down (hem_idx, bt_chunk_num);u64 check_max_num = start_idx + bt_chunk_num;u64 i;for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++) -if (hem[i]) +if (i != hem_idx && hem[i])return false;return true; } -static bool hns_roce_check_bt_null (u64 **bt,u64 start_idx,u32 bt_chunk_num) +static bool hns_roce_check_bt_null (u64 **bt,u64 ba_idx,u32 bt_chunk_num) { +u64 start_idx =round_down (ba_idx, bt_chunk_num);int i;for (i =0 ; i < bt_chunk_num; i++) -if (bt[start_idx + i]) +if (i != ba_idx && bt[start_idx + i])return false;return true;
@@ -723,116 +725,75 @@ int hns_roce_table_get(struct hns_roce_dev *hr_dev,return ret; } +static void clear_mhop_hem (struct hns_roce_dev *hr_dev, +struct hns_roce_hem_table *table,unsigned long obj, +struct hns_roce_hem_mhop *mhop, +struct hns_roce_hem_index *index) +{ +struct ib_device *ibdev = &hr_dev->ib_dev; +u32 hop_num = mhop->hop_num; +u32 chunk_ba_num; +int step_idx; + + index->inited =HEM_INDEX_BUF ; + chunk_ba_num = mhop->bt_chunk_size /BA_BYTE_LEN ; +if (check_whether_bt_num_2 (table->type, hop_num)) { +if (hns_roce_check_hem_null (table->hem, index->buf, + chunk_ba_num, table->num_hem)) + index->inited |=HEM_INDEX_L0 ; + }else if (check_whether_bt_num_3 (table->type, hop_num)) { +if (hns_roce_check_hem_null (table->hem, index->buf, + chunk_ba_num, table->num_hem)) { + index->inited |=HEM_INDEX_L1 ; +if (hns_roce_check_bt_null (table->bt_l1, index->l1, + chunk_ba_num)) + index->inited |=HEM_INDEX_L0 ; + } + } + +if (table->type <HEM_TYPE_MTT ) { +if (hop_num ==HNS_ROCE_HOP_NUM_0 ) + step_idx =0 ; +else + step_idx = hop_num; + +if (hr_dev->hw->clear_hem (hr_dev, table, obj, step_idx)) +ibdev_warn (ibdev,"Clear hop%d HEM failed.\n" , hop_num); + +if (index->inited &HEM_INDEX_L1 ) +if (hr_dev->hw->clear_hem (hr_dev, table, obj,1 )) +ibdev_warn (ibdev,"Clear HEM step 1 failed.\n" ); + +if (index->inited &HEM_INDEX_L0 ) +if (hr_dev->hw->clear_hem (hr_dev, table, obj,0 )) +ibdev_warn (ibdev,"Clear HEM step 0 failed.\n" ); + } +} +static void hns_roce_table_mhop_put (struct hns_roce_dev *hr_dev,struct hns_roce_hem_table *table,unsigned long obj,int check_refcount) { -struct device *dev = hr_dev->dev; -struct hns_roce_hem_mhop mhop; -unsigned long mhop_obj = obj; -u32 bt_chunk_size; -u32 chunk_ba_num; -u32 hop_num; -u32 start_idx; -u32 bt_num; -u64 hem_idx; -u64 bt_l1_idx =0 ; +struct ib_device *ibdev = &hr_dev->ib_dev; +struct hns_roce_hem_index index = {}; +struct hns_roce_hem_mhop mhop = {};int ret; - ret =hns_roce_calc_hem_mhop (hr_dev, table, &mhop_obj, &mhop); -if (ret) -return ; - - bt_chunk_size = mhop.bt_chunk_size; - hop_num = mhop.hop_num; - chunk_ba_num = bt_chunk_size /BA_BYTE_LEN ; - - bt_num =hns_roce_get_bt_num (table->type, hop_num); -switch (bt_num) { -case 3 : - hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + - mhop.l1_idx * chunk_ba_num + mhop.l2_idx; - bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; -break ; -case 2 : - hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; -break ; -case 1 : - hem_idx = mhop.l0_idx; -break ; -default : -dev_err (dev,"Table %d not support hop_num = %d!\n" , - table->type, hop_num); + ret =calc_hem_config (hr_dev, table, obj, &mhop, &index); +if (ret) { +ibdev_err (ibdev,"calc hem config failed!\n" );return ; }mutex_lock (&table->mutex); - -if (check_refcount && (--table->hem[hem_idx]->refcount >0 )) { +if (check_refcount && (--table->hem[index.buf]->refcount >0 )) {mutex_unlock (&table->mutex);return ; } -if (table->type <HEM_TYPE_MTT && hop_num ==1 ) { -if (hr_dev->hw->clear_hem (hr_dev, table, obj,1 )) -dev_warn (dev,"Clear HEM base address failed.\n" ); - }else if (table->type <HEM_TYPE_MTT && hop_num ==2 ) { -if (hr_dev->hw->clear_hem (hr_dev, table, obj,2 )) -dev_warn (dev,"Clear HEM base address failed.\n" ); - }else if (table->type <HEM_TYPE_MTT && - hop_num ==HNS_ROCE_HOP_NUM_0 ) { -if (hr_dev->hw->clear_hem (hr_dev, table, obj,0 )) -dev_warn (dev,"Clear HEM base address failed.\n" ); - } - -/* -* free buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC. -* free bt space chunk for MTT/CQE. -*/ -hns_roce_free_hem (hr_dev, table->hem[hem_idx]); - table->hem[hem_idx] =NULL ; - -if (check_whether_bt_num_2 (table->type, hop_num)) { - start_idx = mhop.l0_idx * chunk_ba_num; -if (hns_roce_check_hem_null (table->hem, start_idx, - chunk_ba_num, table->num_hem)) { -if (table->type <HEM_TYPE_MTT && - hr_dev->hw->clear_hem (hr_dev, table, obj,0 )) -dev_warn (dev,"Clear HEM base address failed.\n" ); - -dma_free_coherent (dev, bt_chunk_size, - table->bt_l0[mhop.l0_idx], - table->bt_l0_dma_addr[mhop.l0_idx]); - table->bt_l0[mhop.l0_idx] =NULL ; - } - }else if (check_whether_bt_num_3 (table->type, hop_num)) { - start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + - mhop.l1_idx * chunk_ba_num; -if (hns_roce_check_hem_null (table->hem, start_idx, - chunk_ba_num, table->num_hem)) { -if (hr_dev->hw->clear_hem (hr_dev, table, obj,1 )) -dev_warn (dev,"Clear HEM base address failed.\n" ); - -dma_free_coherent (dev, bt_chunk_size, - table->bt_l1[bt_l1_idx], - table->bt_l1_dma_addr[bt_l1_idx]); - table->bt_l1[bt_l1_idx] =NULL ; - - start_idx = mhop.l0_idx * chunk_ba_num; -if (hns_roce_check_bt_null (table->bt_l1, start_idx, - chunk_ba_num)) { -if (hr_dev->hw->clear_hem (hr_dev, table, obj, -0 )) -dev_warn (dev,"Clear HEM base address failed.\n" ); - -dma_free_coherent (dev, bt_chunk_size, - table->bt_l0[mhop.l0_idx], - table->bt_l0_dma_addr[mhop.l0_idx]); - table->bt_l0[mhop.l0_idx] =NULL ; - } - } - } +clear_mhop_hem (hr_dev, table, obj, &mhop, &index); +free_mhop_hem (hr_dev, table, &mhop, &index);mutex_unlock (&table->mutex); }
Signature · 1 line
2.8.1
Jason Gunthorpe
AppliedQuoted 7 lines · Weihang Li
Codes about getting/putting HEM(Hardware Entry Memory) in multi-hop addressing are not clear enough, so optimize by spliting and encapsulating them into new functions.
Xi Wang (2):
RDMA/hns: Optimize mhop get flow for multi-hop addressing RDMA/hns: Optimize mhop put flow for multi-hop addressing
Applied to for-next, thanks
Jason