RDMA/erdma: erdma updates 3-22-2023
Subsystems (2)
Cheng Xu
maintainercover RDMA/erdma: erdma updates 3-22-2023
Hi,
This series has some updates for erdma driver:
- #1 unifies the byte ordering API usage in erdma, removes APIs prefixing with underlines and introduces be32_to_cpu_array to copy and swap byte order.
- #2 eliminates unnecessary casting of EQ doorbells.
- #3 refactors the initialization flow to make code cleaner.
Thanks,
Cheng Xu
Cheng Xu (3):
RDMA/erdma: Unify byte ordering APIs usage RDMA/erdma: Eliminate unnecessary casting of EQ doorbells RDMA/erdma: Minor refactor of device init flow
drivers/infiniband/hw/erdma/erdma.h | 2 +- drivers/infiniband/hw/erdma/erdma_cm.h | 10 +++--- drivers/infiniband/hw/erdma/erdma_cmdq.c | 42 ++++-------------------- drivers/infiniband/hw/erdma/erdma_cq.c | 2 +- drivers/infiniband/hw/erdma/erdma_eq.c | 9 +++-- drivers/infiniband/hw/erdma/erdma_main.c | 39 ++++++++++++++++++---- 6 files changed, 51 insertions(+), 53 deletions(-)
Signature · 1 line
2.31.1
Cheng Xu
maintainerPATCH 1/3 RDMA/erdma: Unify byte ordering APIs usage
Replace __be32_to_cpu/__cpu_to_be16 with be32_to_cpu/cpu_to_be16. And use be32_to_cpu_array to copy and swap byte order to hide the loop.
--- drivers/infiniband/hw/erdma/erdma_cm.h | 10 +++++----- drivers/infiniband/hw/erdma/erdma_cmdq.c | 12 +++++------- drivers/infiniband/hw/erdma/erdma_cq.c | 2 +- 3 files changed, 11 insertions(+), 13 deletions(-)
drivers/infiniband/hw/erdma/erdma_cm.h +5 -5
@@ -33,11 +33,11 @@ struct mpa_rr_params {* MPA request/response Hdr bits & fields */ enum { -MPA_RR_FLAG_MARKERS =__cpu_to_be16 (0x8000 ), -MPA_RR_FLAG_CRC =__cpu_to_be16 (0x4000 ), -MPA_RR_FLAG_REJECT =__cpu_to_be16 (0x2000 ), -MPA_RR_RESERVED =__cpu_to_be16 (0x1f00 ), -MPA_RR_MASK_REVISION =__cpu_to_be16 (0x00ff ) +MPA_RR_FLAG_MARKERS =cpu_to_be16 (0x8000 ), +MPA_RR_FLAG_CRC =cpu_to_be16 (0x4000 ), +MPA_RR_FLAG_REJECT =cpu_to_be16 (0x2000 ), +MPA_RR_RESERVED =cpu_to_be16 (0x1f00 ), +MPA_RR_MASK_REVISION =cpu_to_be16 (0x00ff ) };/*
drivers/infiniband/hw/erdma/erdma_cmdq.c +5 -7
@@ -283,7 +283,7 @@ static void *get_next_valid_cmdq_cqe(struct erdma_cmdq *cmdq)__be32 *cqe =get_queue_entry (cmdq->cq.qbuf, cmdq->cq.ci, cmdq->cq.depth,CQE_SHIFT );u32 owner =FIELD_GET (ERDMA_CQE_HDR_OWNER_MASK , -__be32_to_cpu (READ_ONCE (*cqe))); +be32_to_cpu (READ_ONCE (*cqe)));return owner ^ !!(cmdq->cq.ci & cmdq->cq.depth) ? cqe :NULL ; }
@@ -319,7 +319,6 @@ static int erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq)__be32 *cqe;u16 ctx_id;u64 *sqe; -int i; cqe =get_next_valid_cmdq_cqe (cmdq);if (!cqe)
@@ -328,8 +327,8 @@ static int erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq) cmdq->cq.ci++;dma_rmb (); - hdr0 =__be32_to_cpu (*cqe); - sqe_idx =__be32_to_cpu (*(cqe +1 )); + hdr0 =be32_to_cpu (*cqe); + sqe_idx =be32_to_cpu (*(cqe +1 )); sqe =get_queue_entry (cmdq->sq.qbuf, sqe_idx, cmdq->sq.depth,SQEBB_SHIFT );
@@ -341,9 +340,8 @@ static int erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq) comp_wait->cmd_status =ERDMA_CMD_STATUS_FINISHED ; comp_wait->comp_status =FIELD_GET (ERDMA_CQE_HDR_SYNDROME_MASK , hdr0); cmdq->sq.ci += cmdq->sq.wqebb_cnt; - -for (i =0 ; i <4 ; i++) - comp_wait->comp_data[i] =__be32_to_cpu (*(cqe +2 + i)); +/* Copy 16B comp data after cqe hdr to outer */ +be32_to_cpu_array (comp_wait->comp_data, cqe +2 ,4 );if (cmdq->use_event)complete (&comp_wait->wait_event);
drivers/infiniband/hw/erdma/erdma_cq.c +1 -1
@@ -11,7 +11,7 @@ static void *get_next_valid_cqe(struct erdma_cq *cq)__be32 *cqe =get_queue_entry (cq->kern_cq.qbuf, cq->kern_cq.ci, cq->depth,CQE_SHIFT );u32 owner =FIELD_GET (ERDMA_CQE_HDR_OWNER_MASK , -__be32_to_cpu (READ_ONCE (*cqe))); +be32_to_cpu (READ_ONCE (*cqe)));return owner ^ !!(cq->kern_cq.ci & cq->depth) ? cqe :NULL ; }
Signature · 1 line
2.31.1
Cheng Xu
maintainerPATCH 2/3 RDMA/erdma: Eliminate unnecessary casting of EQ doorbells
In reply to Cheng XuUsing void * to define EQ doorbell pointer can eliminate unnecessary casting when performing assignment. Also rename *db_addr* to *db* for a shorter name.
--- drivers/infiniband/hw/erdma/erdma.h | 2 +- drivers/infiniband/hw/erdma/erdma_cmdq.c | 3 +-- drivers/infiniband/hw/erdma/erdma_eq.c | 9 ++++----- 3 files changed, 6 insertions(+), 8 deletions(-)
drivers/infiniband/hw/erdma/erdma.h +1 -1
@@ -32,7 +32,7 @@ struct erdma_eq {atomic64_t event_num;atomic64_t notify_num; -u64 __iomem *db_addr; +void __iomem *db;u64 *db_record; };
drivers/infiniband/hw/erdma/erdma_cmdq.c +1 -2
@@ -166,8 +166,7 @@ static int erdma_cmdq_eq_init(struct erdma_dev *dev)spin_lock_init (&eq->lock);atomic64_set (&eq->event_num,0 ); - eq->db_addr = - (u64 __iomem *)(dev->func_bar +ERDMA_REGS_CEQ_DB_BASE_REG ); + eq->db = dev->func_bar +ERDMA_REGS_CEQ_DB_BASE_REG ; eq->db_record = (u64 *)(eq->qbuf + buf_size); erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_H_REG,
drivers/infiniband/hw/erdma/erdma_eq.c +4 -5
@@ -14,7 +14,7 @@ void notify_eq(struct erdma_eq *eq)FIELD_PREP (ERDMA_EQDB_ARM_MASK ,1 ); *eq->db_record = db_data; -writeq (db_data, eq->db_addr); +writeq (db_data, eq->db);atomic64_inc (&eq->notify_num); }
@@ -98,7 +98,7 @@ int erdma_aeq_init(struct erdma_dev *dev)atomic64_set (&eq->event_num,0 );atomic64_set (&eq->notify_num,0 ); - eq->db_addr = (u64 __iomem *)(dev->func_bar +ERDMA_REGS_AEQ_DB_REG ); + eq->db = dev->func_bar +ERDMA_REGS_AEQ_DB_REG ; eq->db_record = (u64 *)(eq->qbuf + buf_size); erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_H_REG,
@@ -243,9 +243,8 @@ static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn)atomic64_set (&eq->notify_num,0 ); eq->depth =ERDMA_DEFAULT_EQ_DEPTH ; - eq->db_addr = - (u64 __iomem *)(dev->func_bar +ERDMA_REGS_CEQ_DB_BASE_REG + - (ceqn +1 ) *ERDMA_DB_SIZE ); + eq->db = dev->func_bar +ERDMA_REGS_CEQ_DB_BASE_REG + + (ceqn +1 ) *ERDMA_DB_SIZE ; eq->db_record = (u64 *)(eq->qbuf + buf_size); eq->ci =0 ; dev->ceqs[ceqn].dev = dev;
Signature · 1 line
2.31.1
Cheng Xu
maintainerPATCH 3/3 RDMA/erdma: Minor refactor of device init flow
In reply to Cheng XuAfter necessary configuration, driver should wait hardware finishing initialization. The wait sets at CMDQ related function though it has nothing to do with CMDQ. Refactor this part to make code cleaner.
--- drivers/infiniband/hw/erdma/erdma_cmdq.c | 27 +--------------- drivers/infiniband/hw/erdma/erdma_main.c | 39 ++++++++++++++++++++---- 2 files changed, 34 insertions(+), 32 deletions(-)
drivers/infiniband/hw/erdma/erdma_cmdq.c +1 -26
@@ -182,9 +182,8 @@ static int erdma_cmdq_eq_init(struct erdma_dev *dev)int erdma_cmdq_init (struct erdma_dev *dev) { -int err, i;struct erdma_cmdq *cmdq = &dev->cmdq; -u32 sts, ctrl; +int err; cmdq->max_outstandings =ERDMA_CMDQ_MAX_OUTSTANDING ; cmdq->use_event = false;
@@ -207,34 +206,10 @@ int erdma_cmdq_init(struct erdma_dev *dev)if (err) goto err_destroy_cq; - ctrl =FIELD_PREP (ERDMA_REG_DEV_CTRL_INIT_MASK ,1 ); -erdma_reg_write32 (dev,ERDMA_REGS_DEV_CTRL_REG , ctrl); - -for (i =0 ; i <ERDMA_WAIT_DEV_DONE_CNT ; i++) { - sts =erdma_reg_read32_filed (dev,ERDMA_REGS_DEV_ST_REG , -ERDMA_REG_DEV_ST_INIT_DONE_MASK ); -if (sts) -break ; - -msleep (ERDMA_REG_ACCESS_WAIT_MS ); - } - -if (i ==ERDMA_WAIT_DEV_DONE_CNT ) { -dev_err (&dev->pdev->dev,"wait init done failed.\n" ); - err = -ETIMEDOUT ; - goto err_destroy_eq; - } -set_bit (ERDMA_CMDQ_STATE_OK_BIT , &cmdq->state);return 0 ; -err_destroy_eq: -dma_free_coherent (&dev->pdev->dev, - (cmdq->eq.depth <<EQE_SHIFT ) + -ERDMA_EXTRA_BUFFER_SIZE , - cmdq->eq.qbuf, cmdq->eq.qbuf_dma_addr); - err_destroy_cq: dma_free_coherent(&dev->pdev->dev, (cmdq->cq.depth << CQE_SHIFT) +
drivers/infiniband/hw/erdma/erdma_main.c +33 -6
@@ -211,13 +211,36 @@ static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev)return 0 ; } -static void erdma_device_uninit (struct erdma_dev *dev) +static void erdma_hw_reset (struct erdma_dev *dev) {u32 ctrl =FIELD_PREP (ERDMA_REG_DEV_CTRL_RESET_MASK ,1 );erdma_reg_write32 (dev,ERDMA_REGS_DEV_CTRL_REG , ctrl); } +static int erdma_wait_hw_init_done (struct erdma_dev *dev) +{ +int i; + +erdma_reg_write32 (dev,ERDMA_REGS_DEV_CTRL_REG , +FIELD_PREP (ERDMA_REG_DEV_CTRL_INIT_MASK ,1 )); + +for (i =0 ; i <ERDMA_WAIT_DEV_DONE_CNT ; i++) { +if (erdma_reg_read32_filed (dev,ERDMA_REGS_DEV_ST_REG , +ERDMA_REG_DEV_ST_INIT_DONE_MASK )) +break ; + +msleep (ERDMA_REG_ACCESS_WAIT_MS ); + } + +if (i ==ERDMA_WAIT_DEV_DONE_CNT ) { +dev_err (&dev->pdev->dev,"wait init done failed.\n" ); +return -ETIMEDOUT ; + } + +return 0 ; +} +static const struct pci_device_id erdma_pci_tbl[] = { {PCI_DEVICE (PCI_VENDOR_ID_ALIBABA ,0x107f ) }, {}
@@ -293,16 +316,22 @@ static int erdma_probe_dev(struct pci_dev *pdev)if (err) goto err_uninit_aeq; - err =erdma_ceqs_init (dev); + err =erdma_wait_hw_init_done (dev);if (err) goto err_uninit_cmdq; + err =erdma_ceqs_init (dev); +if (err) + goto err_reset_hw; +erdma_finish_cmdq_init (dev);return 0 ; +err_reset_hw: +erdma_hw_reset (dev); + err_uninit_cmdq: -erdma_device_uninit (dev);erdma_cmdq_destroy (dev); err_uninit_aeq:
@@ -334,9 +363,7 @@ static void erdma_remove_dev(struct pci_dev *pdev)struct erdma_dev *dev =pci_get_drvdata (pdev);erdma_ceqs_uninit (dev); - -erdma_device_uninit (dev); - +erdma_hw_reset (dev);erdma_cmdq_destroy (dev);erdma_aeq_destroy (dev);erdma_comm_irq_uninit (dev);
Signature · 1 line
2.31.1
Leon Romanovsky
AppliedQuoted 8 lines · Cheng Xu
This series has some updates for erdma driver:
- #1 unifies the byte ordering API usage in erdma, removes APIs prefixing with underlines and introducesbe32_to_cpu_arrayto copy and swap byte order.
- #2 eliminates unnecessary casting of EQ doorbells.
- #3 refactors the initialization flow to make code cleaner.[...]
Applied, thanks!
[1/3] RDMA/erdma: Unify byte ordering APIs usage
https://git.kernel.org/rdma/rdma/c/de19ec778c7a4e [2/3] RDMA/erdma: Eliminate unnecessary casting of EQ doorbells https://git.kernel.org/rdma/rdma/c/72769dba6dc0b8 [3/3] RDMA/erdma: Minor refactor of device init flow https://git.kernel.org/rdma/rdma/c/901d9d62416bae
Best regards,
Signature · 1 line
Leon Romanovsky <[email protected]>