RDMA/erdma: Introduce hardware statistics support
Subsystems (2)
Cheng Xu
maintainerv3 cover RDMA/erdma: Introduce hardware statistics support
This small patchset introduces the support of hardware statistics. Statistics counters can not be put in CQEs due to limited CQE size. To address this, we provide an extra dma buffer to hardware when posting statistics query request, and then hardware writes back the response to this dma buffer. Based on this, we add the hardware statistics support of erdma.
- #1 introduces dma pool used for hardware responses of CMDQ requests.
- #2 adds hardware statistics support.
Changes in v3:
- Fix compiler warning reported by kernel test robot.
Changes in v2:
- Remove extra layer of erdma_dma_pools_init and erdma_dma_pools_destroy
- Move ERDMA_HW_RESP_SIZE from patch #2 to #1 because it's used in #1
- Remove unrelated change in patch #2
- Remove "hw_" prefix and add "tx_" prefix to some items in erdma_descs
- Use dma_pool_zalloc instead of dma_pool_alloc with __GFP_ZERO flag
- Remove port index check logic in erdma_get_hw_stats
Cheng Xu (2):
RDMA/erdma: Introduce dma pool for hardware responses of CMDQ requests RDMA/erdma: Add hardware statistics support
drivers/infiniband/hw/erdma/erdma.h | 2 + drivers/infiniband/hw/erdma/erdma_hw.h | 39 ++++++++++ drivers/infiniband/hw/erdma/erdma_main.c | 26 ++++++- drivers/infiniband/hw/erdma/erdma_verbs.c | 90 +++++++++++++++++++++++ drivers/infiniband/hw/erdma/erdma_verbs.h | 4 + 5 files changed, 159 insertions(+), 2 deletions(-)
Signature · 1 line
2.31.1
Cheng Xu
maintainerPATCH v3 1/2 RDMA/erdma: Introduce dma pool for hardware responses of CMDQ requests
Hardware response, such as the result of query statistics, may be too long to be directly accommodated within the CQE structure. To address this, we introduce a DMA pool to hold the hardware's responses of CMDQ requests.
--- drivers/infiniband/hw/erdma/erdma.h | 2 ++ drivers/infiniband/hw/erdma/erdma_hw.h | 2 ++ drivers/infiniband/hw/erdma/erdma_main.c | 24 ++++++++++++++++++++++-- 3 files changed, 26 insertions(+), 2 deletions(-)
drivers/infiniband/hw/erdma/erdma.h +2 -0
@@ -212,6 +212,8 @@ struct erdma_dev {atomic_t num_ctx;struct list_head cep_list; + +struct dma_pool *resp_pool; };static inline void *get_queue_entry (void *qbuf,u32 idx,u32 depth,u32 shift)
drivers/infiniband/hw/erdma/erdma_hw.h +2 -0
@@ -357,6 +357,8 @@ struct erdma_cmdq_reflush_req {u32 rq_pi; }; +#define ERDMA_HW_RESP_SIZE 256 +/* cap qword 0 definition */ #define ERDMA_CMD_DEV_CAP_MAX_CQE_MASK GENMASK_ULL(47, 40)#define ERDMA_CMD_DEV_CAP_FLAGS_MASK GENMASK_ULL(31, 24)
drivers/infiniband/hw/erdma/erdma_main.c +22 -2
@@ -172,14 +172,30 @@ static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev) {int ret; + dev->resp_pool =dma_pool_create ("erdma_resp_pool" , &pdev->dev, +ERDMA_HW_RESP_SIZE ,ERDMA_HW_RESP_SIZE , +0 ); +if (!dev->resp_pool) +return -ENOMEM ; + ret =dma_set_mask_and_coherent (&pdev->dev,DMA_BIT_MASK (ERDMA_PCI_WIDTH ));if (ret) -return ret; + goto destroy_pool;dma_set_max_seg_size (&pdev->dev,UINT_MAX );return 0 ; + +destroy_pool: +dma_pool_destroy (dev->resp_pool); + +return ret; +} + +static void erdma_device_uninit (struct erdma_dev *dev) +{ +dma_pool_destroy (dev->resp_pool); }static void erdma_hw_reset (struct erdma_dev *dev)
@@ -273,7 +289,7 @@ static int erdma_probe_dev(struct pci_dev *pdev) err =erdma_request_vectors (dev);if (err) - goto err_iounmap_func_bar; + goto err_uninit_device; err =erdma_comm_irq_init (dev);if (err)
@@ -314,6 +330,9 @@ static int erdma_probe_dev(struct pci_dev *pdev) err_free_vectors:pci_free_irq_vectors (dev->pdev); +err_uninit_device: +erdma_device_uninit (dev); + err_iounmap_func_bar:devm_iounmap (&pdev->dev, dev->func_bar);
@@ -339,6 +358,7 @@ static void erdma_remove_dev(struct pci_dev *pdev)erdma_aeq_destroy (dev);erdma_comm_irq_uninit (dev);pci_free_irq_vectors (dev->pdev); +erdma_device_uninit (dev);devm_iounmap (&pdev->dev, dev->func_bar);pci_release_selected_regions (pdev,ERDMA_BAR_MASK );
Signature · 1 line
2.31.1
Cheng Xu
maintainerPATCH v3 2/2 RDMA/erdma: Add hardware statistics support
In reply to Cheng XuFirst, we add a new command to query hardware statistics, and then implement two functions: ib_device_ops.alloc_hw_port_stats and ib_device_ops.get_hw_stats to allow rdma tool can get the statistics of erdma device.
--- drivers/infiniband/hw/erdma/erdma_hw.h | 37 ++++++++++ drivers/infiniband/hw/erdma/erdma_main.c | 2 + drivers/infiniband/hw/erdma/erdma_verbs.c | 90 +++++++++++++++++++++++ drivers/infiniband/hw/erdma/erdma_verbs.h | 4 + 4 files changed, 133 insertions(+)
drivers/infiniband/hw/erdma/erdma_hw.h +37 -0
@@ -146,6 +146,7 @@ enum CMDQ_COMMON_OPCODE {CMDQ_OPCODE_DESTROY_EQ =1 ,CMDQ_OPCODE_QUERY_FW_INFO =2 ,CMDQ_OPCODE_CONF_MTU =3 , +CMDQ_OPCODE_GET_STATS =4 ,CMDQ_OPCODE_CONF_DEVICE =5 ,CMDQ_OPCODE_ALLOC_DB =8 , CMDQ_OPCODE_FREE_DB = 9,
@@ -359,6 +360,42 @@ struct erdma_cmdq_reflush_req {#define ERDMA_HW_RESP_SIZE 256 +struct erdma_cmdq_query_req { +u64 hdr; +u32 rsvd; +u32 index; + +u64 target_addr; +u32 target_length; +}; + +#define ERDMA_HW_RESP_MAGIC 0x5566 + +struct erdma_cmdq_query_resp_hdr { +u16 magic; +u8 ver; +u8 length; + +u32 index; +u32 rsvd[2 ]; +}; + +struct erdma_cmdq_query_stats_resp { +struct erdma_cmdq_query_resp_hdr hdr; + +u64 tx_req_cnt; +u64 tx_packets_cnt; +u64 tx_bytes_cnt; +u64 tx_drop_packets_cnt; +u64 tx_bps_meter_drop_packets_cnt; +u64 tx_pps_meter_drop_packets_cnt; +u64 rx_packets_cnt; +u64 rx_bytes_cnt; +u64 rx_drop_packets_cnt; +u64 rx_bps_meter_drop_packets_cnt; +u64 rx_pps_meter_drop_packets_cnt; +}; +/* cap qword 0 definition */ #define ERDMA_CMD_DEV_CAP_MAX_CQE_MASK GENMASK_ULL(47, 40)#define ERDMA_CMD_DEV_CAP_FLAGS_MASK GENMASK_ULL(31, 24)
drivers/infiniband/hw/erdma/erdma_main.c +2 -0
@@ -468,6 +468,7 @@ static const struct ib_device_ops erdma_device_ops = { .driver_id = RDMA_DRIVER_ERDMA, .uverbs_abi_ver = ERDMA_ABI_VERSION, + .alloc_hw_port_stats = erdma_alloc_hw_port_stats, .alloc_mr = erdma_ib_alloc_mr, .alloc_pd = erdma_alloc_pd, .alloc_ucontext = erdma_alloc_ucontext,
@@ -479,6 +480,7 @@ static const struct ib_device_ops erdma_device_ops = { .destroy_cq = erdma_destroy_cq, .destroy_qp = erdma_destroy_qp, .get_dma_mr = erdma_get_dma_mr, + .get_hw_stats = erdma_get_hw_stats, .get_port_immutable = erdma_get_port_immutable, .iw_accept = erdma_accept, .iw_add_ref = erdma_qp_get_ref,
drivers/infiniband/hw/erdma/erdma_verbs.c +90 -0
93 lines not on this page · raw message
drivers/infiniband/hw/erdma/erdma_verbs.h +4 -0
@@ -361,5 +361,9 @@ int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset);void erdma_port_event (struct erdma_dev *dev,enum ib_event_type reason);void erdma_set_mtu (struct erdma_dev *dev,u32 mtu); +struct rdma_hw_stats *erdma_alloc_hw_port_stats (struct ib_device *device, +u32 port_num); +int erdma_get_hw_stats (struct ib_device *ibdev,struct rdma_hw_stats *stats, +u32 port,int index);#endif
Signature · 1 line
2.31.1
Leon Romanovsky
AppliedQuoted 8 lines · Cheng Xu
This small patchset introduces the support of hardware statistics. Statistics counters can not be put in CQEs due to limited CQE size. To address this, we provide an extra dma buffer to hardware when posting statistics query request, and then hardware writes back the response to this dma buffer. Based on this, we add the hardware statistics support of erdma.
[...]
Applied, thanks!
[1/2] RDMA/erdma: Introduce dma pool for hardware responses of CMDQ requests
https://git.kernel.org/rdma/rdma/c/68cf9d82f75c07 [2/2] RDMA/erdma: Add hardware statistics support https://git.kernel.org/rdma/rdma/c/63a43a675cb90e
Best regards,
Signature · 1 line
Leon Romanovsky <[email protected]>