Skip to contentSkip to conversation

RDMA/erdma: erdma updates 3-22-2023

PATCH Applied Cheng Xu 2 participants 5 messages Mar 22, 2023

Subsystems (2)
lore
Share & tools
lore

Cheng Xu

author maintainer
#0 lore raw

cover RDMA/erdma: erdma updates 3-22-2023

Hi,

This series has some updates for erdma driver:
- #1 unifies the byte ordering API usage in erdma, removes APIs prefixing with underlines and introduces be32_to_cpu_array to copy and swap byte order.
- #2 eliminates unnecessary casting of EQ doorbells.
- #3 refactors the initialization flow to make code cleaner.

Thanks,
Cheng Xu

Cheng Xu (3):
RDMA/erdma: Unify byte ordering APIs usage RDMA/erdma: Eliminate unnecessary casting of EQ doorbells RDMA/erdma: Minor refactor of device init flow

 drivers/infiniband/hw/erdma/erdma.h      |  2 +-
 drivers/infiniband/hw/erdma/erdma_cm.h   | 10 +++---
 drivers/infiniband/hw/erdma/erdma_cmdq.c | 42 ++++--------------------
 drivers/infiniband/hw/erdma/erdma_cq.c   |  2 +-
 drivers/infiniband/hw/erdma/erdma_eq.c   |  9 +++--
 drivers/infiniband/hw/erdma/erdma_main.c | 39 ++++++++++++++++++----
 6 files changed, 51 insertions(+), 53 deletions(-)
Signature · 1 line
2.31.1
author maintainer
#1 lore raw

PATCH 1/3 RDMA/erdma: Unify byte ordering APIs usage

Replace __be32_to_cpu/__cpu_to_be16 with be32_to_cpu/cpu_to_be16. And use be32_to_cpu_array to copy and swap byte order to hide the loop.

---
 drivers/infiniband/hw/erdma/erdma_cm.h   | 10 +++++-----
 drivers/infiniband/hw/erdma/erdma_cmdq.c | 12 +++++-------
 drivers/infiniband/hw/erdma/erdma_cq.c   |  2 +-
 3 files changed, 11 insertions(+), 13 deletions(-)
drivers/infiniband/hw/erdma/erdma_cm.h +5 -5
index 8a3f998fec9b..a26d80770188 100644
@@ -33,11 +33,11 @@ struct mpa_rr_params {
  * MPA request/response Hdr bits & fields
  */
 enum {
-	MPA_RR_FLAG_MARKERS = __cpu_to_be16(0x8000),
-	MPA_RR_FLAG_CRC = __cpu_to_be16(0x4000),
-	MPA_RR_FLAG_REJECT = __cpu_to_be16(0x2000),
-	MPA_RR_RESERVED = __cpu_to_be16(0x1f00),
-	MPA_RR_MASK_REVISION = __cpu_to_be16(0x00ff)
+	MPA_RR_FLAG_MARKERS = cpu_to_be16(0x8000),
+	MPA_RR_FLAG_CRC = cpu_to_be16(0x4000),
+	MPA_RR_FLAG_REJECT = cpu_to_be16(0x2000),
+	MPA_RR_RESERVED = cpu_to_be16(0x1f00),
+	MPA_RR_MASK_REVISION = cpu_to_be16(0x00ff)
 };
 
 /*
drivers/infiniband/hw/erdma/erdma_cmdq.c +5 -7
index 6ebfa6989b11..3fd33b85c15a 100644
@@ -283,7 +283,7 @@ static void *get_next_valid_cmdq_cqe(struct erdma_cmdq *cmdq)
 	__be32 *cqe = get_queue_entry(cmdq->cq.qbuf, cmdq->cq.ci,
 				      cmdq->cq.depth, CQE_SHIFT);
 	u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
-			      __be32_to_cpu(READ_ONCE(*cqe)));
+			      be32_to_cpu(READ_ONCE(*cqe)));
 
 	return owner ^ !!(cmdq->cq.ci & cmdq->cq.depth) ? cqe : NULL;
 }
@@ -319,7 +319,6 @@ static int erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq)
 	__be32 *cqe;
 	u16 ctx_id;
 	u64 *sqe;
-	int i;
 
 	cqe = get_next_valid_cmdq_cqe(cmdq);
 	if (!cqe)
@@ -328,8 +327,8 @@ static int erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq)
 	cmdq->cq.ci++;
 
 	dma_rmb();
-	hdr0 = __be32_to_cpu(*cqe);
-	sqe_idx = __be32_to_cpu(*(cqe + 1));
+	hdr0 = be32_to_cpu(*cqe);
+	sqe_idx = be32_to_cpu(*(cqe + 1));
 
 	sqe = get_queue_entry(cmdq->sq.qbuf, sqe_idx, cmdq->sq.depth,
 			      SQEBB_SHIFT);
@@ -341,9 +340,8 @@ static int erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq)
 	comp_wait->cmd_status = ERDMA_CMD_STATUS_FINISHED;
 	comp_wait->comp_status = FIELD_GET(ERDMA_CQE_HDR_SYNDROME_MASK, hdr0);
 	cmdq->sq.ci += cmdq->sq.wqebb_cnt;
-
-	for (i = 0; i < 4; i++)
-		comp_wait->comp_data[i] = __be32_to_cpu(*(cqe + 2 + i));
+	/* Copy 16B comp data after cqe hdr to outer */
+	be32_to_cpu_array(comp_wait->comp_data, cqe + 2, 4);
 
 	if (cmdq->use_event)
 		complete(&comp_wait->wait_event);
drivers/infiniband/hw/erdma/erdma_cq.c +1 -1
index cabd8678b355..7e2bfa6ed86f 100644
@@ -11,7 +11,7 @@ static void *get_next_valid_cqe(struct erdma_cq *cq)
 	__be32 *cqe = get_queue_entry(cq->kern_cq.qbuf, cq->kern_cq.ci,
 				      cq->depth, CQE_SHIFT);
 	u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
-			      __be32_to_cpu(READ_ONCE(*cqe)));
+			      be32_to_cpu(READ_ONCE(*cqe)));
 
 	return owner ^ !!(cq->kern_cq.ci & cq->depth) ? cqe : NULL;
 }
Signature · 1 line
2.31.1
author maintainer
#2 lore raw

PATCH 2/3 RDMA/erdma: Eliminate unnecessary casting of EQ doorbells

In reply to Cheng Xu

Using void * to define EQ doorbell pointer can eliminate unnecessary casting when performing assignment. Also rename *db_addr* to *db* for a shorter name.

---
 drivers/infiniband/hw/erdma/erdma.h      | 2 +-
 drivers/infiniband/hw/erdma/erdma_cmdq.c | 3 +--
 drivers/infiniband/hw/erdma/erdma_eq.c   | 9 ++++-----
 3 files changed, 6 insertions(+), 8 deletions(-)
drivers/infiniband/hw/erdma/erdma.h +1 -1
index 3d8c11aa23a2..e819e4032490 100644
@@ -32,7 +32,7 @@ struct erdma_eq {
 	atomic64_t event_num;
 	atomic64_t notify_num;
 
-	u64 __iomem *db_addr;
+	void __iomem *db;
 	u64 *db_record;
 };
 
drivers/infiniband/hw/erdma/erdma_cmdq.c +1 -2
index 3fd33b85c15a..eb6aaf7e28f5 100644
@@ -166,8 +166,7 @@ static int erdma_cmdq_eq_init(struct erdma_dev *dev)
 	spin_lock_init(&eq->lock);
 	atomic64_set(&eq->event_num, 0);
 
-	eq->db_addr =
-		(u64 __iomem *)(dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG);
+	eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG;
 	eq->db_record = (u64 *)(eq->qbuf + buf_size);
 
 	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_H_REG,
drivers/infiniband/hw/erdma/erdma_eq.c +4 -5
index ed54130d924b..ea47cb21fdb8 100644
@@ -14,7 +14,7 @@ void notify_eq(struct erdma_eq *eq)
 		      FIELD_PREP(ERDMA_EQDB_ARM_MASK, 1);
 
 	*eq->db_record = db_data;
-	writeq(db_data, eq->db_addr);
+	writeq(db_data, eq->db);
 
 	atomic64_inc(&eq->notify_num);
 }
@@ -98,7 +98,7 @@ int erdma_aeq_init(struct erdma_dev *dev)
 	atomic64_set(&eq->event_num, 0);
 	atomic64_set(&eq->notify_num, 0);
 
-	eq->db_addr = (u64 __iomem *)(dev->func_bar + ERDMA_REGS_AEQ_DB_REG);
+	eq->db = dev->func_bar + ERDMA_REGS_AEQ_DB_REG;
 	eq->db_record = (u64 *)(eq->qbuf + buf_size);
 
 	erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_H_REG,
@@ -243,9 +243,8 @@ static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn)
 	atomic64_set(&eq->notify_num, 0);
 
 	eq->depth = ERDMA_DEFAULT_EQ_DEPTH;
-	eq->db_addr =
-		(u64 __iomem *)(dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG +
-				(ceqn + 1) * ERDMA_DB_SIZE);
+	eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG +
+		 (ceqn + 1) * ERDMA_DB_SIZE;
 	eq->db_record = (u64 *)(eq->qbuf + buf_size);
 	eq->ci = 0;
 	dev->ceqs[ceqn].dev = dev;
Signature · 1 line
2.31.1
author maintainer
#3 lore raw

PATCH 3/3 RDMA/erdma: Minor refactor of device init flow

In reply to Cheng Xu

After necessary configuration, driver should wait hardware finishing initialization. The wait sets at CMDQ related function though it has nothing to do with CMDQ. Refactor this part to make code cleaner.

---
 drivers/infiniband/hw/erdma/erdma_cmdq.c | 27 +---------------
 drivers/infiniband/hw/erdma/erdma_main.c | 39 ++++++++++++++++++++----
 2 files changed, 34 insertions(+), 32 deletions(-)
drivers/infiniband/hw/erdma/erdma_cmdq.c +1 -26
index eb6aaf7e28f5..a151a7bdd504 100644
@@ -182,9 +182,8 @@ static int erdma_cmdq_eq_init(struct erdma_dev *dev)
 
 int erdma_cmdq_init(struct erdma_dev *dev)
 {
-	int err, i;
 	struct erdma_cmdq *cmdq = &dev->cmdq;
-	u32 sts, ctrl;
+	int err;
 
 	cmdq->max_outstandings = ERDMA_CMDQ_MAX_OUTSTANDING;
 	cmdq->use_event = false;
@@ -207,34 +206,10 @@ int erdma_cmdq_init(struct erdma_dev *dev)
 	if (err)
 		goto err_destroy_cq;
 
-	ctrl = FIELD_PREP(ERDMA_REG_DEV_CTRL_INIT_MASK, 1);
-	erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, ctrl);
-
-	for (i = 0; i < ERDMA_WAIT_DEV_DONE_CNT; i++) {
-		sts = erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG,
-					     ERDMA_REG_DEV_ST_INIT_DONE_MASK);
-		if (sts)
-			break;
-
-		msleep(ERDMA_REG_ACCESS_WAIT_MS);
-	}
-
-	if (i == ERDMA_WAIT_DEV_DONE_CNT) {
-		dev_err(&dev->pdev->dev, "wait init done failed.\n");
-		err = -ETIMEDOUT;
-		goto err_destroy_eq;
-	}
-
 	set_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);
 
 	return 0;
 
-err_destroy_eq:
-	dma_free_coherent(&dev->pdev->dev,
-			  (cmdq->eq.depth << EQE_SHIFT) +
-				  ERDMA_EXTRA_BUFFER_SIZE,
-			  cmdq->eq.qbuf, cmdq->eq.qbuf_dma_addr);
-
 err_destroy_cq:
 	dma_free_coherent(&dev->pdev->dev,
 			  (cmdq->cq.depth << CQE_SHIFT) +
drivers/infiniband/hw/erdma/erdma_main.c +33 -6
index 5dc31e5df5cb..2c8fd00b0816 100644
@@ -211,13 +211,36 @@ static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev)
 	return 0;
 }
 
-static void erdma_device_uninit(struct erdma_dev *dev)
+static void erdma_hw_reset(struct erdma_dev *dev)
 {
 	u32 ctrl = FIELD_PREP(ERDMA_REG_DEV_CTRL_RESET_MASK, 1);
 
 	erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, ctrl);
 }
 
+static int erdma_wait_hw_init_done(struct erdma_dev *dev)
+{
+	int i;
+
+	erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG,
+			  FIELD_PREP(ERDMA_REG_DEV_CTRL_INIT_MASK, 1));
+
+	for (i = 0; i < ERDMA_WAIT_DEV_DONE_CNT; i++) {
+		if (erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG,
+					   ERDMA_REG_DEV_ST_INIT_DONE_MASK))
+			break;
+
+		msleep(ERDMA_REG_ACCESS_WAIT_MS);
+	}
+
+	if (i == ERDMA_WAIT_DEV_DONE_CNT) {
+		dev_err(&dev->pdev->dev, "wait init done failed.\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
 static const struct pci_device_id erdma_pci_tbl[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_ALIBABA, 0x107f) },
 	{}
@@ -293,16 +316,22 @@ static int erdma_probe_dev(struct pci_dev *pdev)
 	if (err)
 		goto err_uninit_aeq;
 
-	err = erdma_ceqs_init(dev);
+	err = erdma_wait_hw_init_done(dev);
 	if (err)
 		goto err_uninit_cmdq;
 
+	err = erdma_ceqs_init(dev);
+	if (err)
+		goto err_reset_hw;
+
 	erdma_finish_cmdq_init(dev);
 
 	return 0;
 
+err_reset_hw:
+	erdma_hw_reset(dev);
+
 err_uninit_cmdq:
-	erdma_device_uninit(dev);
 	erdma_cmdq_destroy(dev);
 
 err_uninit_aeq:
@@ -334,9 +363,7 @@ static void erdma_remove_dev(struct pci_dev *pdev)
 	struct erdma_dev *dev = pci_get_drvdata(pdev);
 
 	erdma_ceqs_uninit(dev);
-
-	erdma_device_uninit(dev);
-
+	erdma_hw_reset(dev);
 	erdma_cmdq_destroy(dev);
 	erdma_aeq_destroy(dev);
 	erdma_comm_irq_uninit(dev);
Signature · 1 line
2.31.1
#4 lore raw
In reply to Cheng Xu
Quoted 8 lines · Cheng Xu

This series has some updates for erdma driver:
- #1 unifies the byte ordering API usage in erdma, removes APIs prefixing with underlines and introduces be32_to_cpu_array to copy and swap byte order.
- #2 eliminates unnecessary casting of EQ doorbells.
- #3 refactors the initialization flow to make code cleaner.

[...]

Applied, thanks!

[1/3] RDMA/erdma: Unify byte ordering APIs usage

      https://git.kernel.org/rdma/rdma/c/de19ec778c7a4e
[2/3] RDMA/erdma: Eliminate unnecessary casting of EQ doorbells
      https://git.kernel.org/rdma/rdma/c/72769dba6dc0b8
[3/3] RDMA/erdma: Minor refactor of device init flow
      https://git.kernel.org/rdma/rdma/c/901d9d62416bae

Best regards,

Signature · 1 line
Leon Romanovsky <[email protected]>