Elastic RDMA Adapter (ERDMA) driver

index 3072e5d6b692..7dd56210226f 100644
@@ -250,6 +250,7 @@ enum rdma_driver_id {
 	RDMA_DRIVER_QIB,
 	RDMA_DRIVER_EFA,
 	RDMA_DRIVER_SIW,
+	RDMA_DRIVER_ERDMA,
 };
 
 enum ib_uverbs_gid_type {

new file mode 100644
index 000000000000..cd5683b04078
@@ -0,0 +1,474 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/*
+ * Authors: Cheng Xu <[email protected]>
+ *          Kai Shen <[email protected]>
+ * Copyright (c) 2020-2021, Alibaba Group.
+ */
+
+#ifndef __ERDMA_HW_H__
+#define __ERDMA_HW_H__
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+
+/* PCIe device related definition. */
+#define PCI_VENDOR_ID_ALIBABA 0x1ded
+
+#define ERDMA_FUNC_BAR     0
+#define ERDMA_MISX_BAR     2
+
+#define ERDMA_BAR_MASK (BIT(ERDMA_FUNC_BAR) | BIT(ERDMA_MISX_BAR))
+
+/* MSI-X related. */
+#define ERDMA_NUM_MSIX_VEC        32
+#define ERDMA_MSIX_VECTOR_CMDQ    0
+
+/* PCIe Bar0 Registers. */
+#define ERDMA_REGS_VERSION_REG             0x0
+#define ERDMA_REGS_DEV_CTRL_REG            0x10
+#define ERDMA_REGS_DEV_ST_REG              0x14
+#define ERDMA_REGS_NETDEV_MAC_L_REG        0x18
+#define ERDMA_REGS_NETDEV_MAC_H_REG        0x1C
+#define ERDMA_REGS_CMDQ_SQ_ADDR_L_REG      0x20
+#define ERDMA_REGS_CMDQ_SQ_ADDR_H_REG      0x24
+#define ERDMA_REGS_CMDQ_CQ_ADDR_L_REG      0x28
+#define ERDMA_REGS_CMDQ_CQ_ADDR_H_REG      0x2C
+#define ERDMA_REGS_CMDQ_DEPTH_REG          0x30
+#define ERDMA_REGS_CMDQ_EQ_DEPTH_REG       0x34
+#define ERDMA_REGS_CMDQ_EQ_ADDR_L_REG      0x38
+#define ERDMA_REGS_CMDQ_EQ_ADDR_H_REG      0x3C
+#define ERDMA_REGS_AEQ_ADDR_L_REG          0x40
+#define ERDMA_REGS_AEQ_ADDR_H_REG          0x44
+#define ERDMA_REGS_AEQ_DEPTH_REG           0x48
+#define ERDMA_REGS_GRP_NUM_REG             0x4c
+#define ERDMA_REGS_AEQ_DB_REG              0x50
+#define ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG     0x60
+#define ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG     0x68
+#define ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG     0x70
+#define ERDMA_AEQ_DB_HOST_ADDR_REG         0x78
+#define ERDMA_REGS_CEQ_DB_BASE_REG         0x100
+#define ERDMA_CMDQ_SQDB_REG                0x200
+#define ERDMA_CMDQ_CQDB_REG                0x300
+
+/* DEV_CTRL_REG details. */
+#define ERDMA_REG_DEV_CTRL_RESET_MASK       0x00000001
+#define ERDMA_REG_DEV_CTRL_INIT_MASK        0x00000002
+
+/* DEV_ST_REG details. */
+#define ERDMA_REG_DEV_ST_RESET_DONE_MASK    0x00000001U
+#define ERDMA_REG_DEV_ST_INIT_DONE_MASK     0x00000002U
+
+/* eRDMA PCIe DBs definition. */
+#define ERDMA_BAR_DB_SPACE_BASE     4096
+
+#define ERDMA_BAR_SQDB_SPACE_OFFSET ERDMA_BAR_DB_SPACE_BASE
+#define ERDMA_BAR_SQDB_SPACE_SIZE   (384 * 1024)
+
+#define ERDMA_BAR_RQDB_SPACE_OFFSET (ERDMA_BAR_SQDB_SPACE_OFFSET + ERDMA_BAR_SQDB_SPACE_SIZE)
+#define ERDMA_BAR_RQDB_SPACE_SIZE   (96 * 1024)
+
+#define ERDMA_BAR_CQDB_SPACE_OFFSET (ERDMA_BAR_RQDB_SPACE_OFFSET + ERDMA_BAR_RQDB_SPACE_SIZE)
+
+/* Doorbell page resources related. */
+/*
+ * Max # of parallelly issued directSQE is 3072 per device,
+ * hardware organizes this into 24 group, per group has 128 credits.
+ */
+#define ERDMA_DWQE_MAX_GRP_CNT          24
+#define ERDMA_DWQE_NUM_PER_GRP          128
+
+#define ERDMA_DWQE_TYPE0_CNT            64
+#define ERDMA_DWQE_TYPE1_CNT            496
+#define ERDMA_DWQE_TYPE1_CNT_PER_PAGE   16  /* type1 DB contains 2 DBs, takes 256Byte. */
+
+#define ERDMA_SDB_SHARED_PAGE_INDEX     95
+
+/* Doorbell related. */
+#define ERDMA_CQDB_EQN_MASK              GENMASK_ULL(63, 56)
+#define ERDMA_CQDB_CQN_MASK              GENMASK_ULL(55, 32)
+#define ERDMA_CQDB_ARM_MASK              BIT_ULL(31)
+#define ERDMA_CQDB_SOL_MASK              BIT_ULL(30)
+#define ERDMA_CQDB_CMDSN_MASK            GENMASK_ULL(29, 28)
+#define ERDMA_CQDB_CI_MASK               GENMASK_ULL(23, 0)
+
+#define ERDMA_EQDB_ARM_MASK              BIT(31)
+#define ERDMA_EQDB_CI_MASK               GENMASK_ULL(23, 0)
+
+/* WQE related. */
+#define EQE_SIZE 16
+#define EQE_SHIFT 4
+#define RQE_SIZE 32
+#define RQE_SHIFT 5
+#define CQE_SIZE 32
+#define CQE_SHIFT 5
+#define SQEBB_SIZE 32
+#define SQEBB_SHIFT 5
+#define SQEBB_MASK (~(SQEBB_SIZE - 1))
+#define SQEBB_ALIGN(size) ((size + SQEBB_SIZE - 1) & SQEBB_MASK)
+#define SQEBB_COUNT(size) (SQEBB_ALIGN(size) >> SQEBB_SHIFT)
+
+#define ERDMA_MAX_SQE_SIZE 128
+#define ERDMA_MAX_WQEBB_PER_SQE 4
+
+/* CMDQ related. */
+#define ERDMA_CMDQ_MAX_OUTSTANDING      128
+#define ERDMA_CMDQ_SQE_SIZE             64
+
+/* cmdq sub module definition. */
+enum CMDQ_WQE_SUB_MOD {
+	CMDQ_SUBMOD_RDMA    = 0,
+	CMDQ_SUBMOD_COMMON  = 1
+};
+
+enum CMDQ_RDMA_OPCODE {
+	CMDQ_OPCODE_QUERY_DEVICE = 0,
+	CMDQ_OPCODE_CREATE_QP    = 1,
+	CMDQ_OPCODE_DESTROY_QP   = 2,
+	CMDQ_OPCODE_MODIFY_QP    = 3,
+	CMDQ_OPCODE_CREATE_CQ    = 4,
+	CMDQ_OPCODE_DESTROY_CQ   = 5,
+	CMDQ_OPCODE_REG_MR       = 8,
+	CMDQ_OPCODE_DEREG_MR     = 9
+};
+
+enum CMDQ_COMMON_OPCODE {
+	CMDQ_OPCODE_CREATE_EQ  = 0,
+	CMDQ_OPCODE_DESTROY_EQ = 1
+};
+
+/* cmdq-SQE HDR */
+#define ERDMA_CMD_HDR_WQEBB_CNT_MASK     GENMASK_ULL(54, 52)
+#define ERDMA_CMD_HDR_CONTEXT_COOKIE     GENMASK_ULL(47, 32)
+#define ERDMA_CMD_HDR_SUB_MOD_MASK       GENMASK_ULL(25, 24)
+#define ERDMA_CMD_HDR_OPCODE_MASK        GENMASK_ULL(23, 16)
+#define ERDMA_CMD_HDR_WQEBB_INDEX_MASK   GENMASK_ULL(15, 0)
+
+struct erdma_cmdq_destroy_cq_req {
+	u64 hdr;
+	u32 cqn;
+};
+
+struct erdma_cmdq_create_eq_req {
+	u64 hdr;
+	u64 qbuf_addr;
+	u8  vector_idx;
+	u8  eqn;
+	u8  depth;
+	u8  qtype;
+	u32 db_dma_addr_l;
+	u32 db_dma_addr_h;
+};
+
+struct erdma_cmdq_destroy_eq_req {
+	u64 hdr;
+	u64 rsvd0;
+	u8  vector_idx;
+	u8  eqn;
+	u8  rsvd1;
+	u8  qtype;
+};
+
+/* create_cq cfg0 */
+#define ERDMA_CMD_CREATE_CQ_DEPTH_MASK      GENMASK(31, 24)
+#define ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK   GENMASK(23, 20)
+#define ERDMA_CMD_CREATE_CQ_CQN_MASK        GENMASK(19, 0)
+
+/* create_cq cfg1 */
+#define ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK    GENMASK(31, 16)
+#define ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK   BIT(15)
+#define ERDMA_CMD_CREATE_CQ_EQN_MASK        GENMASK(9, 0)
+
+struct erdma_cmdq_create_cq_req {
+	u64 hdr;
+	u32 cfg0;
+	u32 qbuf_addr_l;
+	u32 qbuf_addr_h;
+	u32 cfg1;
+	u64 cq_db_info_addr;
+	u32 first_page_offset;
+};
+
+/* regmr/deregmr cfg0 */
+#define ERDMA_CMD_MR_VALID_MASK    BIT(31)
+#define ERDMA_CMD_MR_KEY_MASK      GENMASK(27, 20)
+#define ERDMA_CMD_MR_MPT_IDX_MASK  GENMASK(19, 0)
+
+/* regmr cfg1 */
+#define ERDMA_CMD_REGMR_PD_MASK       GENMASK(31, 12)
+#define ERDMA_CMD_REGMR_TYPE_MASK     GENMASK(7, 6)
+#define ERDMA_CMD_REGMR_RIGHT_MASK    GENMASK(5, 2)
+#define ERDMA_CMD_REGMR_ACC_MODE_MASK GENMASK(1, 0)
+
+/* regmr cfg2 */
+#define ERDMA_CMD_REGMR_PAGESIZE_MASK GENMASK(31, 27)
+#define ERDMA_CMD_REGMR_MTT_TYPE_MASK GENMASK(21, 20)
+#define ERDMA_CMD_REGMR_MTT_CNT_MASK  GENMASK(19, 0)
+
+struct erdma_cmdq_reg_mr_req {
+	u64 hdr;
+	u32 cfg0;
+	u32 cfg1;
+	u64 start_va;
+	u32 size;
+	u32 cfg2;
+	u64 phy_addr[4];
+};
+
+struct erdma_cmdq_dereg_mr_req {
+	u64 hdr;
+	u32 cfg0;
+};
+
+/* modify qp cfg0 */
+#define ERDMA_CMD_MODIFY_QP_STATE_MASK  GENMASK(31, 24)
+#define ERDMA_CMD_MODIFY_QP_CC_MASK     GENMASK(23, 20)
+#define ERDMA_CMD_MODIFY_QP_QPN_MASK    GENMASK(19, 0)
+
+struct erdma_cmdq_modify_qp_req {
+	u64 hdr;
+	u32 cfg0;
+	u32 remote_qpn;
+	u32 dip;
+	u32 sip;
+	u16 sport;
+	u16 dport;
+	u32 send_nxt;
+	u32 recv_nxt;
+};
+
+/* create qp cfg0 */
+#define ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK   GENMASK(31, 20)
+#define ERDMA_CMD_CREATE_QP_QPN_MASK        GENMASK(19, 0)
+
+/* create qp cfg1 */
+#define ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK   GENMASK(31, 20)
+#define ERDMA_CMD_CREATE_QP_PD_MASK         GENMASK(19, 0)
+
+/* cqn_mtt_cfg */
+#define ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK  GENMASK(31, 28)
+#define ERDMA_CMD_CREATE_QP_CQN_MASK        GENMASK(23, 0)
+
+/* mtt_cfg */
+#define ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK GENMASK(31, 12)
+#define ERDMA_CMD_CREATE_QP_MTT_CNT_MASK     GENMASK(11, 1)
+#define ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK    BIT(0)
+
+struct erdma_cmdq_create_qp_req {
+	u64 hdr;
+	u32 cfg0;
+	u32 cfg1;
+	u32 sq_cqn_mtt_cfg;
+	u32 rq_cqn_mtt_cfg;
+	u64 sq_buf_addr;
+	u64 rq_buf_addr;
+	u32 sq_mtt_cfg;
+	u32 rq_mtt_cfg;
+	u64 sq_db_info_dma_addr;
+	u64 rq_db_info_dma_addr;
+};
+
+struct erdma_cmdq_destroy_qp_req {
+	u64 hdr;
+	u32 qpn;
+};
+
+#define ERDMA_CMD_DEV_CAP0_MAX_CQE_MASK      GENMASK_ULL(47, 40)
+#define ERDMA_CMD_DEV_CAP0_MAX_RECV_WR_MASK  GENMASK_ULL(23, 16)
+#define ERDMA_CMD_DEV_CAP0_MAX_MR_SIZE_MASK  GENMASK_ULL(7, 0)
+
+#define ERDMA_CMD_DEV_CAP1_DMA_LOCAL_KEY_MASK GENMASK_ULL(63, 32)
+#define ERDMA_CMD_DEV_CAP1_DEFAULT_CC_MASK    GENMASK_ULL(31, 28)
+#define ERDMA_CMD_DEV_CAP1_QBLOCK_MASK        GENMASK_ULL(27, 16)
+#define ERDMA_CMD_DEV_CAP1_MAX_MW_MASK        GENMASK_ULL(7, 0)
+
+#define ERDMA_NQP_PER_QBLOCK 1024
+
+/* CQE hdr */
+#define ERDMA_CQE_HDR_OWNER_MASK         BIT(31)
+#define ERDMA_CQE_HDR_OPCODE_MASK        GENMASK(23, 16)
+#define ERDMA_CQE_HDR_QTYPE_MASK         GENMASK(15, 8)
+#define ERDMA_CQE_HDR_SYNDROME_MASK      GENMASK(7, 0)
+
+#define ERDMA_CQE_QTYPE_SQ    0
+#define ERDMA_CQE_QTYPE_RQ    1
+#define ERDMA_CQE_QTYPE_CMDQ  2
+
+struct erdma_cqe {
+	__be32 hdr;
+	__be32 qe_idx;
+	__be32 qpn;
+	__be32 imm_data;
+	__be32 size;
+	__be32 rsvd[3];
+};
+
+struct erdma_sge {
+	__aligned_le64 laddr;
+	__le32         length;
+	__le32         lkey;
+};
+
+/* Receive Queue Element */
+struct erdma_rqe {
+	__le16 qe_idx;
+	__le16 rsvd;
+	__le32 qpn;
+	__le32 rsvd2;
+	__le32 rsvd3;
+	__le64 to;
+	__le32 length;
+	__le32 stag;
+};
+
+/* SQE */
+#define ERDMA_SQE_HDR_SGL_LEN_MASK       GENMASK_ULL(63, 56)
+#define ERDMA_SQE_HDR_WQEBB_CNT_MASK     GENMASK_ULL(54, 52)
+#define ERDMA_SQE_HDR_QPN_MASK           GENMASK_ULL(51, 32)
+#define ERDMA_SQE_HDR_OPCODE_MASK        GENMASK_ULL(31, 27)
+#define ERDMA_SQE_HDR_DWQE_MASK          BIT_ULL(26)
+#define ERDMA_SQE_HDR_INLINE_MASK        BIT_ULL(25)
+#define ERDMA_SQE_HDR_FENCE_MASK         BIT_ULL(24)
+#define ERDMA_SQE_HDR_SE_MASK            BIT_ULL(23)
+#define ERDMA_SQE_HDR_CE_MASK            BIT_ULL(22)
+#define ERDMA_SQE_HDR_WQEBB_INDEX_MASK   GENMASK_ULL(15, 0)
+
+/* REG MR attrs */
+#define ERDMA_SQE_MR_ACCESS_MODE_MASK	GENMASK_ULL(1, 0)
+#define ERDMA_SQE_MR_ACCESS_RIGHT_MASK	GENMASK_ULL(5, 2)
+#define ERDMA_SQE_MR_MTT_TYPE_MASK		GENMASK_ULL(7, 6)
+#define ERDMA_SQE_MR_MTT_COUNT_MASK		GENMASK_ULL(31, 12)
+
+struct erdma_write_sqe {
+	__le64 hdr;
+	__le32 imm_data;
+	__le32 length;
+
+	__le32 sink_stag;
+	__le32 sink_to_low;
+	__le32 sink_to_high;
+
+	__le32 rsvd;
+
+	struct erdma_sge sgl[0];
+};
+
+struct erdma_send_sqe {
+	__le64 hdr;
+	__le32 imm_data;
+	__le32 length;
+	struct erdma_sge sgl[0];
+};
+
+struct erdma_readreq_sqe {
+	__le64 hdr;
+	__le32 invalid_stag;
+	__le32 length;
+	__le32 sink_stag;
+	__le32 sink_to_low;
+	__le32 sink_to_high;
+	__le32 rsvd0;
+};
+
+struct erdma_reg_mr_sqe {
+	__le64 hdr;
+	__le64 addr;
+	__le32 length;
+	__le32 stag;
+	__le32 attrs;
+	__le32 reserved;
+	__u64 inline_addr[0];
+};
+
+/* EQ related. */
+#define ERDMA_DEFAULT_EQ_DEPTH 256
+
+/* ceqe */
+#define ERDMA_CEQE_HDR_DB_MASK  BIT_ULL(63)
+#define ERDMA_CEQE_HDR_PI_MASK  GENMASK_ULL(55, 32)
+#define ERDMA_CEQE_HDR_O_MASK   BIT_ULL(31)
+#define ERDMA_CEQE_HDR_CQN_MASK GENMASK_ULL(19, 0)
+
+/* aeqe */
+#define ERDMA_AEQE_HDR_O_MASK       BIT(31)
+#define ERDMA_AEQE_HDR_TYPE_MASK    GENMASK(23, 16)
+#define ERDMA_AEQE_HDR_SUBTYPE_MASK GENMASK(7, 0)
+
+#define ERDMA_AE_TYPE_QP_FATAL_EVENT     0
+#define ERDMA_AE_TYPE_QP_ERQ_ERR_EVENT   1
+#define ERDMA_AE_TYPE_ACC_ERR_EVENT      2
+#define ERDMA_AE_TYPE_CQ_ERR             3
+#define ERDMA_AE_TYPE_OTHER_ERROR        4
+
+struct erdma_aeqe {
+	__le32   hdr;
+	__le32   event_data0;
+	__le32   event_data1;
+	__le32   rsvd2;
+};
+
+enum erdma_opcode {
+	ERDMA_OP_WRITE           = 0,
+	ERDMA_OP_READ            = 1,
+	ERDMA_OP_SEND            = 2,
+	ERDMA_OP_SEND_WITH_IMM   = 3,
+
+	ERDMA_OP_RECEIVE         = 4,
+	ERDMA_OP_RECV_IMM        = 5,
+	ERDMA_OP_RECV_INV        = 6,
+
+	ERDMA_OP_REQ_ERR         = 7,
+	ERDNA_OP_READ_RESPONSE   = 8,
+	ERDMA_OP_WRITE_WITH_IMM  = 9,
+
+	ERDMA_OP_RECV_ERR        = 10,
+
+	ERDMA_OP_INVALIDATE     = 11,
+	ERDMA_OP_RSP_SEND_IMM   = 12,
+	ERDMA_OP_SEND_WITH_INV  = 13,
+
+	ERDMA_OP_REG_MR         = 14,
+	ERDMA_OP_LOCAL_INV      = 15,
+	ERDMA_OP_READ_WITH_INV  = 16,
+	ERDMA_NUM_OPCODES       = 17,
+	ERDMA_OP_INVALID        = ERDMA_NUM_OPCODES + 1
+};
+
+enum erdma_wc_status {
+	ERDMA_WC_SUCCESS = 0,
+	ERDMA_WC_GENERAL_ERR = 1,
+	ERDMA_WC_RECV_WQE_FORMAT_ERR = 2,
+	ERDMA_WC_RECV_STAG_INVALID_ERR = 3,
+	ERDMA_WC_RECV_ADDR_VIOLATION_ERR = 4,
+	ERDMA_WC_RECV_RIGHT_VIOLATION_ERR = 5,
+	ERDMA_WC_RECV_PDID_ERR = 6,
+	ERDMA_WC_RECV_WARRPING_ERR = 7,
+	ERDMA_WC_SEND_WQE_FORMAT_ERR = 8,
+	ERDMA_WC_SEND_WQE_ORD_EXCEED = 9,
+	ERDMA_WC_SEND_STAG_INVALID_ERR = 10,
+	ERDMA_WC_SEND_ADDR_VIOLATION_ERR = 11,
+	ERDMA_WC_SEND_RIGHT_VIOLATION_ERR = 12,
+	ERDMA_WC_SEND_PDID_ERR = 13,
+	ERDMA_WC_SEND_WARRPING_ERR = 14,
+	ERDMA_WC_FLUSH_ERR = 15,
+	ERDMA_WC_RETRY_EXC_ERR = 16,
+	ERDMA_NUM_WC_STATUS
+};
+
+enum erdma_vendor_err {
+	ERDMA_WC_VENDOR_NO_ERR = 0,
+	ERDMA_WC_VENDOR_INVALID_RQE = 1,
+	ERDMA_WC_VENDOR_RQE_INVALID_STAG = 2,
+	ERDMA_WC_VENDOR_RQE_ADDR_VIOLATION = 3,
+	ERDMA_WC_VENDOR_RQE_ACCESS_RIGHT_ERR = 4,
+	ERDMA_WC_VENDOR_RQE_INVALID_PD = 5,
+	ERDMA_WC_VENDOR_RQE_WRAP_ERR = 6,
+	ERDMA_WC_VENDOR_INVALID_SQE = 0x20,
+	ERDMA_WC_VENDOR_ZERO_ORD = 0x21,
+	ERDMA_WC_VENDOR_SQE_INVALID_STAG = 0x30,
+	ERDMA_WC_VENDOR_SQE_ADDR_VIOLATION = 0x31,
+	ERDMA_WC_VENDOR_SQE_ACCESS_ERR = 0x32,
+	ERDMA_WC_VENDOR_SQE_INVALID_PD = 0x33,
+	ERDMA_WC_VENDOR_SQE_WARP_ERR = 0x34
+};
+
+#endif

new file mode 100644
index 000000000000..fd0fee698874
@@ -0,0 +1,381 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/*
+ * Authors: Cheng Xu <[email protected]>
+ *          Kai Shen <[email protected]>
+ * Copyright (c) 2020-2021, Alibaba Group.
+ */
+
+#ifndef __ERDMA_H__
+#define __ERDMA_H__
+
+#include <linux/bitfield.h>
+#include <linux/netdevice.h>
+#include <linux/xarray.h>
+#include <rdma/ib_verbs.h>
+
+#include "erdma_hw.h"
+
+#define DRV_MODULE_NAME "erdma"
+
+struct erdma_eq {
+	void *qbuf;
+	dma_addr_t qbuf_dma_addr;
+
+	u32 depth;
+	u64 __iomem *db_addr;
+
+	spinlock_t lock;
+
+	u16 ci;
+	u16 owner;
+
+	atomic64_t event_num;
+	atomic64_t notify_num;
+
+	void *db_info;
+};
+
+struct erdma_cmdq_sq {
+	void *qbuf;
+	dma_addr_t qbuf_dma_addr;
+
+	spinlock_t lock;
+	u64 __iomem *db_addr;
+
+	u16 ci;
+	u16 pi;
+
+	u16 depth;
+	u16 wqebb_cnt;
+
+	void *db_info;
+
+	u64 total_cmds;
+	u64 total_comp_cmds;
+};
+
+struct erdma_cmdq_cq {
+	void *qbuf;
+
+	dma_addr_t qbuf_dma_addr;
+
+	u64 __iomem *db_addr;
+	spinlock_t lock;
+
+	u32 ci;
+	u16 owner;
+	u16 depth;
+
+	void *db_info;
+
+	atomic64_t cq_armed_num;
+};
+
+enum {
+	ERDMA_CMD_STATUS_INIT,
+	ERDMA_CMD_STATUS_ISSUED,
+	ERDMA_CMD_STATUS_FINISHED,
+	ERDMA_CMD_STATUS_TIMEOUT
+};
+
+struct erdma_comp_wait {
+	struct completion wait_event;
+	u32 cmd_status;
+	u32 ctx_id;
+	u16 sq_pi;
+	u8 comp_status;
+	u8 rsvd;
+	u32 comp_data[4];
+};
+
+enum {
+	ERDMA_CMDQ_STATE_OK_BIT = 0,
+	ERDMA_CMDQ_STATE_TIMEOUT_BIT = 1,
+	ERDMA_CMDQ_STATE_CTX_ERR_BIT = 2,
+};
+
+#define ERDMA_CMDQ_TIMEOUT_MS       15000
+#define ERDMA_REG_ACCESS_WAIT_MS    20
+#define ERDMA_WAIT_DEV_DONE_CNT     500
+
+struct erdma_cmdq {
+	void *dev;
+
+	unsigned long *comp_wait_bitmap;
+	struct erdma_comp_wait *wait_pool;
+	spinlock_t lock;
+
+	u8 use_event;
+
+	struct erdma_cmdq_sq sq;
+	struct erdma_cmdq_cq cq;
+	struct erdma_eq eq;
+
+	unsigned long state;
+
+	struct semaphore credits;
+	u16 max_outstandings;
+};
+
+struct erdma_devattr {
+	unsigned int device;
+	unsigned int version;
+
+	u32 vendor_id;
+	u32 vendor_part_id;
+	u32 sw_version;
+	u32 max_qp;
+	u32 max_send_wr;
+	u32 max_recv_wr;
+	u32 max_ord;
+	u32 max_ird;
+
+	enum ib_device_cap_flags cap_flags;
+	u32 max_send_sge;
+	u32 max_recv_sge;
+	u32 max_sge_rd;
+	u32 max_cq;
+	u32 max_cqe;
+	u64 max_mr_size;
+	u32 max_mr;
+	u32 max_pd;
+	u32 max_mw;
+	u32 max_srq;
+	u32 max_srq_wr;
+	u32 max_srq_sge;
+	u32 local_dma_key;
+};
+
+#define ERDMA_IRQNAME_SIZE 50
+struct erdma_irq_info {
+	char name[ERDMA_IRQNAME_SIZE];
+	irq_handler_t handler;
+	u32 msix_vector;
+	void *data;
+	int cpu;
+	cpumask_t affinity_hint_mask;
+};
+
+struct erdma_eq_cb {
+	u8 ready;
+	u8 rsvd[3];
+	void *dev;
+	struct erdma_irq_info irq_info;
+	struct erdma_eq eq;
+	struct tasklet_struct tasklet;
+};
+
+#define COMPROMISE_CC ERDMA_CC_CUBIC
+enum erdma_cc_method {
+	ERDMA_CC_NEWRENO = 0,
+	ERDMA_CC_CUBIC,
+	ERDMA_CC_HPCC_RTT,
+	ERDMA_CC_HPCC_ECN,
+	ERDMA_CC_HPCC_INT,
+	ERDMA_CC_METHODS_NUM
+};
+
+struct erdma_resource_cb {
+	unsigned long *bitmap;
+	spinlock_t lock;
+	u32 next_alloc_idx;
+	u32 max_cap;
+};
+
+enum {
+	ERDMA_RES_TYPE_PD = 0,
+	ERDMA_RES_TYPE_STAG_IDX = 1,
+	ERDMA_RES_CNT = 2,
+};
+
+static inline int erdma_alloc_idx(struct erdma_resource_cb *res_cb)
+{
+	int idx;
+	unsigned long flags;
+	u32 start_idx = res_cb->next_alloc_idx;
+
+	spin_lock_irqsave(&res_cb->lock, flags);
+	idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap, start_idx);
+	if (idx == res_cb->max_cap) {
+		idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap);
+		if (idx == res_cb->max_cap) {
+			res_cb->next_alloc_idx = 1;
+			spin_unlock_irqrestore(&res_cb->lock, flags);
+			return -ENOSPC;
+		}
+	}
+
+	set_bit(idx, res_cb->bitmap);
+	spin_unlock_irqrestore(&res_cb->lock, flags);
+	res_cb->next_alloc_idx = idx + 1;
+	return idx;
+}
+
+static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx)
+{
+	unsigned long flags;
+	u32 used;
+
+	spin_lock_irqsave(&res_cb->lock, flags);
+	used = test_and_clear_bit(idx, res_cb->bitmap);
+	spin_unlock_irqrestore(&res_cb->lock, flags);
+	WARN_ON(!used);
+}
+
+#define ERDMA_EXTRA_BUFFER_SIZE 8
+
+struct erdma_dev {
+	struct ib_device ibdev;
+	struct net_device *netdev;
+	struct pci_dev *pdev;
+
+	struct notifier_block netdev_nb;
+	unsigned char peer_addr[MAX_ADDR_LEN];
+
+	/* physical port state (only one port per device) */
+	enum ib_port_state state;
+
+	u8 __iomem *func_bar;
+
+	resource_size_t func_bar_addr;
+	resource_size_t func_bar_len;
+
+	u32 dma_width;
+
+	struct erdma_irq_info comm_irq;
+	struct erdma_cmdq cmdq;
+
+	u16 irq_num;
+	u16 rsvd;
+
+	struct erdma_eq_cb aeq;
+	struct erdma_eq_cb ceqs[31];
+
+	struct erdma_devattr attrs;
+
+	spinlock_t lock;
+
+	struct erdma_resource_cb res_cb[ERDMA_RES_CNT];
+	struct xarray qp_xa;
+	struct xarray cq_xa;
+
+	u32 next_alloc_qpn;
+	u32 next_alloc_cqn;
+
+	spinlock_t db_bitmap_lock;
+
+	/* We provide 64 uContexts that each has one SQ doorbell Page. */
+	DECLARE_BITMAP(sdb_page, ERDMA_DWQE_TYPE0_CNT);
+	/* We provide 496 uContexts that each has one SQ normal Db, and one directWQE db */
+	DECLARE_BITMAP(sdb_entry, ERDMA_DWQE_TYPE1_CNT);
+
+	atomic_t num_pd;
+	atomic_t num_qp;
+	atomic_t num_cq;
+	atomic_t num_cep;
+	atomic_t num_mr;
+	atomic_t num_ctx;
+
+	struct list_head cep_list;
+
+	u32 is_registered;
+	struct dentry *debugfs;
+
+	int numa_node;
+	int cc_method;
+	int grp_num;
+	int disable_dwqe;
+	int dwqe_pages;
+	int dwqe_entries;
+};
+
+static inline struct erdma_dev *to_edev(struct ib_device *ibdev)
+{
+	return container_of(ibdev, struct erdma_dev, ibdev);
+}
+
+static inline u32 erdma_reg_read32(struct erdma_dev *dev, u32 reg)
+{
+	return readl(dev->func_bar + reg);
+}
+
+static inline u64 erdma_reg_read64(struct erdma_dev *dev, u32 reg)
+{
+	return readq(dev->func_bar + reg);
+}
+
+static inline void erdma_reg_write32(struct erdma_dev *dev, u32 reg, u32 value)
+{
+	writel(value, dev->func_bar + reg);
+}
+
+static inline void erdma_reg_write64(struct erdma_dev *dev, u32 reg, u64 value)
+{
+	writeq(value, dev->func_bar + reg);
+}
+
+static inline u32 erdma_reg_read32_filed(struct erdma_dev *dev, u32 reg, u32 filed_mask)
+{
+	u32 val = erdma_reg_read32(dev, reg);
+
+	return FIELD_GET(filed_mask, val);
+}
+
+static inline int erdma_poll_ceq_event(struct erdma_eq *ceq)
+{
+	__le64 *ceqe;
+	u16 queue_size_mask = ceq->depth - 1;
+	u64 val;
+
+	ceqe = ceq->qbuf + ((ceq->ci & queue_size_mask) << EQE_SHIFT);
+
+	val = READ_ONCE(*ceqe);
+	if (FIELD_GET(ERDMA_CEQE_HDR_O_MASK, val) == ceq->owner) {
+		dma_rmb();
+		ceq->ci++;
+
+		if ((ceq->ci & queue_size_mask) == 0)
+			ceq->owner = !ceq->owner;
+
+		atomic64_inc(&ceq->event_num);
+
+		return FIELD_GET(ERDMA_CEQE_HDR_CQN_MASK, val);
+	}
+
+	return -1;
+}
+
+static inline void notify_eq(struct erdma_eq *eq)
+{
+	u64 db_data = FIELD_PREP(ERDMA_EQDB_CI_MASK, eq->ci) |
+		FIELD_PREP(ERDMA_EQDB_ARM_MASK, 1);
+
+	*(u64 *)eq->db_info = db_data;
+	writeq(db_data, eq->db_addr);
+
+	atomic64_inc(&eq->notify_num);
+}
+
+int erdma_cmdq_init(struct erdma_dev *dev);
+void erdma_finish_cmdq_init(struct erdma_dev *dev);
+void erdma_cmdq_destroy(struct erdma_dev *dev);
+
+#define ERDMA_CMDQ_BUILD_REQ_HDR(hdr, mod, op) \
+do { \
+	*(u64 *)(hdr) = FIELD_PREP(ERDMA_CMD_HDR_SUB_MOD_MASK, mod); \
+	*(u64 *)(hdr) |= FIELD_PREP(ERDMA_CMD_HDR_OPCODE_MASK, op); \
+} while (0)
+
+int erdma_post_cmd_wait(struct erdma_cmdq *cmdq, u64 *req, u32 req_size, u64 *resp0, u64 *resp1);
+void erdma_cmdq_completion_handler(struct erdma_dev *dev);
+
+int erdma_ceqs_init(struct erdma_dev *dev);
+void erdma_ceqs_uninit(struct erdma_dev *dev);
+
+int erdma_aeq_init(struct erdma_dev *dev);
+void erdma_aeq_destroy(struct erdma_dev *dev);
+
+void erdma_aeq_event_handler(struct erdma_dev *dev);
+void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb);
+
+#endif

new file mode 100644
index 000000000000..3eca524fca24
@@ -0,0 +1,489 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Authors: Cheng Xu <[email protected]>
+ *          Kai Shen <[email protected]>
+ * Copyright (c) 2020-2021, Alibaba Group.
+ */
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+
+#include "erdma.h"
+#include "erdma_hw.h"
+#include "erdma_verbs.h"
+
+static inline void arm_cmdq_cq(struct erdma_cmdq *cmdq)
+{
+	u64 db_data = FIELD_PREP(ERDMA_CQDB_CI_MASK, cmdq->cq.ci) |
+		FIELD_PREP(ERDMA_CQDB_ARM_MASK, 1);
+
+	*(u64 *)cmdq->cq.db_info = db_data;
+	writeq(db_data, cmdq->cq.db_addr);
+
+	atomic64_inc(&cmdq->cq.cq_armed_num);
+}
+
+static inline void kick_cmdq_db(struct erdma_cmdq *cmdq)
+{
+	u64 db_data = FIELD_PREP(ERDMA_CMD_HDR_WQEBB_INDEX_MASK, cmdq->sq.pi);
+
+	*(u64 *)cmdq->sq.db_info = db_data;
+	writeq(db_data, cmdq->sq.db_addr);
+}
+
+
+static struct erdma_comp_wait *get_comp_wait(struct erdma_cmdq *cmdq)
+{
+	int comp_idx;
+
+	spin_lock(&cmdq->lock);
+	comp_idx = find_first_zero_bit(cmdq->comp_wait_bitmap, cmdq->max_outstandings);
+	if (comp_idx == cmdq->max_outstandings) {
+		spin_unlock(&cmdq->lock);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	set_bit(comp_idx, cmdq->comp_wait_bitmap);
+	spin_unlock(&cmdq->lock);
+
+	return &cmdq->wait_pool[comp_idx];
+}
+
+static void put_comp_wait(struct erdma_cmdq *cmdq, struct erdma_comp_wait *comp_wait)
+{
+	int used;
+
+	cmdq->wait_pool[comp_wait->ctx_id].cmd_status = ERDMA_CMD_STATUS_INIT;
+	spin_lock(&cmdq->lock);
+	used = test_and_clear_bit(comp_wait->ctx_id, cmdq->comp_wait_bitmap);
+	spin_unlock(&cmdq->lock);
+
+	WARN_ON(!used);
+}
+
+static int erdma_cmdq_wait_res_init(struct erdma_dev *dev, struct erdma_cmdq *cmdq)
+{
+	int i;
+
+	cmdq->wait_pool = devm_kcalloc(&dev->pdev->dev, cmdq->max_outstandings,
+		sizeof(struct erdma_comp_wait), GFP_KERNEL);
+	if (!cmdq->wait_pool)
+		return -ENOMEM;
+
+	spin_lock_init(&cmdq->lock);
+	cmdq->comp_wait_bitmap = devm_kcalloc(&dev->pdev->dev,
+		BITS_TO_LONGS(cmdq->max_outstandings), sizeof(unsigned long), GFP_KERNEL);
+	if (!cmdq->comp_wait_bitmap) {
+		devm_kfree(&dev->pdev->dev, cmdq->wait_pool);
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < cmdq->max_outstandings; i++) {
+		init_completion(&cmdq->wait_pool[i].wait_event);
+		cmdq->wait_pool[i].ctx_id = i;
+	}
+
+	return 0;
+}
+
+static int erdma_cmdq_sq_init(struct erdma_dev *dev)
+{
+	struct erdma_cmdq *cmdq = &dev->cmdq;
+	struct erdma_cmdq_sq *sq = &cmdq->sq;
+	u32 buf_size;
+
+	sq->wqebb_cnt = SQEBB_COUNT(ERDMA_CMDQ_SQE_SIZE);
+	sq->depth = cmdq->max_outstandings * sq->wqebb_cnt;
+
+	buf_size = sq->depth << SQEBB_SHIFT;
+
+	sq->qbuf = dma_alloc_coherent(&dev->pdev->dev, buf_size + ERDMA_EXTRA_BUFFER_SIZE,
+		&sq->qbuf_dma_addr, GFP_KERNEL);
+	if (!sq->qbuf)
+		return -ENOMEM;
+
+	sq->db_info = sq->qbuf + buf_size;
+
+	spin_lock_init(&sq->lock);
+
+	sq->ci = 0;
+	sq->pi = 0;
+	sq->total_cmds = 0;
+	sq->total_comp_cmds = 0;
+
+	sq->db_addr = (u64 __iomem *)(dev->func_bar + ERDMA_CMDQ_SQDB_REG);
+
+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_SQ_ADDR_H_REG, upper_32_bits(sq->qbuf_dma_addr));
+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_SQ_ADDR_L_REG, lower_32_bits(sq->qbuf_dma_addr));
+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_DEPTH_REG, sq->depth);
+	erdma_reg_write64(dev, ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG, sq->qbuf_dma_addr + buf_size);
+
+	return 0;
+}
+
+static int erdma_cmdq_cq_init(struct erdma_dev *dev)
+{
+	struct erdma_cmdq *cmdq = &dev->cmdq;
+	struct erdma_cmdq_cq *cq = &cmdq->cq;
+	u32 buf_size;
+
+	cq->depth = cmdq->max_outstandings * 2;
+	buf_size = cq->depth << CQE_SHIFT;
+
+	cq->qbuf = dma_alloc_coherent(&dev->pdev->dev, buf_size + ERDMA_EXTRA_BUFFER_SIZE,
+			&cq->qbuf_dma_addr, GFP_KERNEL);
+	if (!cq->qbuf)
+		return -ENOMEM;
+
+	cq->db_info = cq->qbuf + buf_size;
+
+	memset(cq->qbuf, 0, buf_size + ERDMA_EXTRA_BUFFER_SIZE);
+
+	spin_lock_init(&cq->lock);
+
+	cq->db_addr = (u64 __iomem *)(dev->func_bar + ERDMA_CMDQ_CQDB_REG);
+	cq->ci = 0;
+	cq->owner = 1;
+
+	atomic64_set(&cq->cq_armed_num, 0);
+
+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_CQ_ADDR_H_REG, upper_32_bits(cq->qbuf_dma_addr));
+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_CQ_ADDR_L_REG, lower_32_bits(cq->qbuf_dma_addr));
+	erdma_reg_write64(dev, ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG, cq->qbuf_dma_addr + buf_size);
+
+	return 0;
+}
+
+static int erdma_cmdq_eq_init(struct erdma_dev *dev)
+{
+	struct erdma_cmdq *cmdq = &dev->cmdq;
+	struct erdma_eq *eq = &cmdq->eq;
+	u32 buf_size;
+
+	eq->depth = cmdq->max_outstandings;
+	buf_size = eq->depth << EQE_SHIFT;
+
+	eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, buf_size + ERDMA_EXTRA_BUFFER_SIZE,
+			&eq->qbuf_dma_addr, GFP_KERNEL);
+	if (!eq->qbuf)
+		return -ENOMEM;
+
+	eq->db_info = eq->qbuf + buf_size;
+
+	memset(eq->qbuf, 0, buf_size);
+	memset(eq->db_info, 0, 8);
+
+	spin_lock_init(&eq->lock);
+	atomic64_set(&eq->event_num, 0);
+
+	eq->db_addr = (u64 __iomem *)(dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG);
+	eq->ci = 0;
+	eq->owner = 1;
+
+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_H_REG, upper_32_bits(eq->qbuf_dma_addr));
+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_L_REG, lower_32_bits(eq->qbuf_dma_addr));
+	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_DEPTH_REG, eq->depth);
+	erdma_reg_write64(dev, ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG, eq->qbuf_dma_addr + buf_size);
+
+	return 0;
+}
+
+int erdma_cmdq_init(struct erdma_dev *dev)
+{
+	int err, i;
+	struct erdma_cmdq *cmdq = &dev->cmdq;
+	u32 status, ctrl;
+
+	cmdq->max_outstandings = ERDMA_CMDQ_MAX_OUTSTANDING;
+	cmdq->dev = dev;
+	cmdq->use_event = 0;
+
+	sema_init(&cmdq->credits, cmdq->max_outstandings);
+
+	err = erdma_cmdq_wait_res_init(dev, cmdq);
+	if (err)
+		return err;
+
+	err = erdma_cmdq_sq_init(dev);
+	if (err)
+		return err;
+
+	err = erdma_cmdq_cq_init(dev);
+	if (err)
+		goto err_destroy_sq;
+
+	err = erdma_cmdq_eq_init(dev);
+	if (err)
+		goto err_destroy_cq;
+
+	ctrl = FIELD_PREP(ERDMA_REG_DEV_CTRL_INIT_MASK, 1);
+	erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, ctrl);
+
+	for (i = 0; i < ERDMA_WAIT_DEV_DONE_CNT; i++) {
+		status = erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG,
+			ERDMA_REG_DEV_ST_INIT_DONE_MASK);
+		if (status)
+			break;
+
+		msleep(ERDMA_REG_ACCESS_WAIT_MS);
+	}
+
+	if (i == ERDMA_WAIT_DEV_DONE_CNT) {
+		dev_err(&dev->pdev->dev, "wait init done failed.\n");
+		err = -ETIMEDOUT;
+		goto err_destroy_eq;
+	}
+
+	set_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);
+
+	return 0;
+
+err_destroy_eq:
+	dma_free_coherent(&dev->pdev->dev,
+		(cmdq->eq.depth << EQE_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE,
+		cmdq->eq.qbuf, cmdq->eq.qbuf_dma_addr);
+
+err_destroy_cq:
+	dma_free_coherent(&dev->pdev->dev,
+		(cmdq->cq.depth << CQE_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE,
+		cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr);
+
+err_destroy_sq:
+	dma_free_coherent(&dev->pdev->dev,
+		(cmdq->sq.depth << SQEBB_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE,
+		cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr);
+
+	return err;
+}
+
+void erdma_finish_cmdq_init(struct erdma_dev *dev)
+{
+	/* after device init successfully, change cmdq to event mode. */
+	dev->cmdq.use_event = true;
+	arm_cmdq_cq(&dev->cmdq);
+}
+
+void erdma_cmdq_destroy(struct erdma_dev *dev)
+{
+	struct erdma_cmdq *cmdq = &dev->cmdq;
+
+	clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);
+
+	dma_free_coherent(&dev->pdev->dev,
+		(cmdq->eq.depth << EQE_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE,
+		cmdq->eq.qbuf, cmdq->eq.qbuf_dma_addr);
+	dma_free_coherent(&dev->pdev->dev,
+		(cmdq->sq.depth << SQEBB_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE,
+		cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr);
+	dma_free_coherent(&dev->pdev->dev,
+		(cmdq->cq.depth << CQE_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE,
+		cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr);
+}
+
+static inline void *get_cmdq_sqe(struct erdma_cmdq *cmdq, u16 idx)
+{
+	idx &= (cmdq->sq.depth - 1);
+	return cmdq->sq.qbuf + (idx << SQEBB_SHIFT);
+}
+
+static inline void *get_cmdq_cqe(struct erdma_cmdq *cmdq, u16 idx)
+{
+	idx &= (cmdq->cq.depth - 1);
+	return cmdq->cq.qbuf + (idx << CQE_SHIFT);
+}
+
+static void push_cmdq_sqe(struct erdma_cmdq *cmdq, u64 *req,
+			 size_t req_len, struct erdma_comp_wait *comp_wait)
+{
+	__le64 *wqe;
+	u64 hdr = *req;
+
+	comp_wait->cmd_status = ERDMA_CMD_STATUS_ISSUED;
+	reinit_completion(&comp_wait->wait_event);
+	comp_wait->sq_pi = cmdq->sq.pi;
+
+	wqe = get_cmdq_sqe(cmdq, cmdq->sq.pi);
+	memcpy(wqe, req, req_len);
+
+	cmdq->sq.pi += cmdq->sq.wqebb_cnt;
+	hdr |= FIELD_PREP(ERDMA_CMD_HDR_WQEBB_INDEX_MASK, cmdq->sq.pi);
+	hdr |= FIELD_PREP(ERDMA_CMD_HDR_CONTEXT_COOKIE, comp_wait->ctx_id);
+	hdr |= FIELD_PREP(ERDMA_CMD_HDR_WQEBB_CNT_MASK, cmdq->sq.wqebb_cnt - 1);
+	*wqe = hdr;
+
+	cmdq->sq.total_cmds++;
+
+	kick_cmdq_db(cmdq);
+}
+
+static void erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq, __be32 *cqe)
+{
+	struct erdma_comp_wait *comp_wait;
+	u16 sqe_idx, ctx_id;
+	u64 *sqe;
+	int i;
+	u32 hdr0 = __be32_to_cpu(*cqe);
+
+	sqe_idx = __be32_to_cpu(*(cqe + 1));
+	sqe = (u64 *)get_cmdq_sqe(cmdq, sqe_idx);
+
+	ctx_id = FIELD_GET(ERDMA_CMD_HDR_CONTEXT_COOKIE, *sqe);
+	comp_wait = &cmdq->wait_pool[ctx_id];
+	if (comp_wait->cmd_status != ERDMA_CMD_STATUS_ISSUED)
+		return;
+
+	comp_wait->cmd_status = ERDMA_CMD_STATUS_FINISHED;
+	comp_wait->comp_status = FIELD_GET(ERDMA_CQE_HDR_SYNDROME_MASK, hdr0);
+	cmdq->sq.ci += cmdq->sq.wqebb_cnt;
+
+	for (i = 0; i < 4; i++)
+		comp_wait->comp_data[i] = __be32_to_cpu(*(cqe + 2 + i));
+
+	if (cmdq->use_event)
+		complete(&comp_wait->wait_event);
+}
+
+static void erdma_polling_cmd_completions(struct erdma_cmdq *cmdq)
+{
+	u32 hdr;
+	__be32 *cqe;
+	unsigned long flags;
+	u16 comp_num = 0;
+	u8 owner, expect_owner;
+	u16 cqe_idx;
+
+	spin_lock_irqsave(&cmdq->cq.lock, flags);
+
+	expect_owner = cmdq->cq.owner;
+	cqe_idx = cmdq->cq.ci & (cmdq->cq.depth - 1);
+
+	while (1) {
+		cqe = (__be32 *)get_cmdq_cqe(cmdq, cqe_idx);
+		hdr = __be32_to_cpu(READ_ONCE(*cqe));
+
+		owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK, hdr);
+		if (owner != expect_owner)
+			break;
+
+		dma_rmb();
+		erdma_poll_single_cmd_completion(cmdq, cqe);
+		comp_num++;
+		if (cqe_idx == cmdq->cq.depth - 1) {
+			cqe_idx = 0;
+			expect_owner = !expect_owner;
+		} else {
+			cqe_idx++;
+		}
+	}
+
+	if (comp_num) {
+		cmdq->cq.ci += comp_num;
+		cmdq->cq.owner = expect_owner;
+		cmdq->sq.total_comp_cmds += comp_num;
+
+		if (cmdq->use_event)
+			arm_cmdq_cq(cmdq);
+	}
+
+	spin_unlock_irqrestore(&cmdq->cq.lock, flags);
+}
+
+void erdma_cmdq_completion_handler(struct erdma_dev *dev)
+{
+	int cqn, got_event = 0;
+
+	if (!test_bit(ERDMA_CMDQ_STATE_OK_BIT, &dev->cmdq.state) || !dev->cmdq.use_event)
+		return;
+
+	while ((cqn = erdma_poll_ceq_event(&dev->cmdq.eq)) != -1)
+		got_event++;
+
+	if (got_event)
+		erdma_polling_cmd_completions(&dev->cmdq);
+
+	notify_eq(&dev->cmdq.eq);
+}
+
+static int erdma_poll_cmd_completion(struct erdma_comp_wait *comp_ctx,
+				     struct erdma_cmdq *cmdq, u32 timeout)
+{
+	unsigned long comp_timeout = jiffies + msecs_to_jiffies(timeout);
+
+	while (1) {
+		erdma_polling_cmd_completions(cmdq);
+		if (comp_ctx->cmd_status != ERDMA_CMD_STATUS_ISSUED)
+			break;
+
+		if (time_is_before_jiffies(comp_timeout))
+			return -ETIME;
+
+		msleep(20);
+	}
+
+	return 0;
+}
+
+static int erdma_wait_cmd_completion(struct erdma_comp_wait *comp_ctx,
+				     struct erdma_cmdq *cmdq, u32 timeout)
+{
+	unsigned long flags = 0;
+
+	wait_for_completion_timeout(&comp_ctx->wait_event,
+		msecs_to_jiffies(timeout));
+
+	if (unlikely(comp_ctx->cmd_status != ERDMA_CMD_STATUS_FINISHED)) {
+		spin_lock_irqsave(&cmdq->cq.lock, flags);
+		comp_ctx->cmd_status = ERDMA_CMD_STATUS_TIMEOUT;
+		spin_unlock_irqrestore(&cmdq->cq.lock, flags);
+		return -ETIME;
+	}
+
+	return 0;
+}
+
+int erdma_post_cmd_wait(struct erdma_cmdq *cmdq, u64 *req, u32 req_size, u64 *resp0, u64 *resp1)
+{
+	struct erdma_comp_wait *comp_wait;
+	int ret;
+
+	if (!test_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state))
+		return -ENODEV;
+
+	down(&cmdq->credits);
+
+	comp_wait = get_comp_wait(cmdq);
+	if (IS_ERR(comp_wait)) {
+		clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);
+		set_bit(ERDMA_CMDQ_STATE_CTX_ERR_BIT, &cmdq->state);
+		up(&cmdq->credits);
+		return PTR_ERR(comp_wait);
+	}
+
+	spin_lock(&cmdq->sq.lock);
+	push_cmdq_sqe(cmdq, req, req_size, comp_wait);
+	spin_unlock(&cmdq->sq.lock);
+
+	if (cmdq->use_event)
+		ret = erdma_wait_cmd_completion(comp_wait, cmdq, ERDMA_CMDQ_TIMEOUT_MS);
+	else
+		ret = erdma_poll_cmd_completion(comp_wait, cmdq, ERDMA_CMDQ_TIMEOUT_MS);
+
+	if (ret) {
+		set_bit(ERDMA_CMDQ_STATE_TIMEOUT_BIT, &cmdq->state);
+		clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);
+		goto out;
+	}
+
+	ret = comp_wait->comp_status;
+
+	if (resp0 && resp1) {
+		*resp0 = *((u64 *)&comp_wait->comp_data[0]);
+		*resp1 = *((u64 *)&comp_wait->comp_data[2]);
+	}
+	put_comp_wait(cmdq, comp_wait);
+
+out:
+	up(&cmdq->credits);
+
+	return ret;
+}

new file mode 100644
index 000000000000..67dae3a6a245
@@ -0,0 +1,346 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Authors: Cheng Xu <[email protected]>
+ *          Kai Shen <[email protected]>
+ * Copyright (c) 2020-2021, Alibaba Group.
+ */
+
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <linux/pci.h>
+
+#include <rdma/iw_cm.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_user_verbs.h>
+
+#include "erdma.h"
+#include "erdma_cm.h"
+#include "erdma_hw.h"
+#include "erdma_verbs.h"
+
+static inline int
+erdma_poll_aeq_event(struct erdma_eq *aeq, void *out)
+{
+	struct erdma_aeqe *aeqe = (struct erdma_aeqe *)aeq->qbuf + (aeq->ci & (aeq->depth - 1));
+	u32 val;
+
+	val = le32_to_cpu(READ_ONCE(aeqe->hdr));
+	if (FIELD_GET(ERDMA_AEQE_HDR_O_MASK, val) == aeq->owner) {
+		dma_rmb();
+		aeq->ci++;
+		if ((aeq->ci & (aeq->depth - 1)) == 0)
+			aeq->owner = !aeq->owner;
+
+		atomic64_inc(&aeq->event_num);
+		if (out)
+			memcpy(out, aeqe, sizeof(struct erdma_aeqe));
+
+		return 1;
+	}
+
+	return 0;
+}
+
+void erdma_aeq_event_handler(struct erdma_dev *dev)
+{
+	struct erdma_aeqe aeqe;
+	u32 cqn, qpn;
+	struct erdma_qp *qp;
+	struct erdma_cq *cq;
+	struct ib_event event;
+
+	memset(&event, 0, sizeof(event));
+	while (erdma_poll_aeq_event(&dev->aeq.eq, &aeqe)) {
+		if (FIELD_GET(ERDMA_AEQE_HDR_TYPE_MASK, aeqe.hdr) == ERDMA_AE_TYPE_CQ_ERR) {
+			cqn = aeqe.event_data0;
+			cq = find_cq_by_cqn(dev, cqn);
+			if (!cq)
+				continue;
+			event.device = cq->ibcq.device;
+			event.element.cq = &cq->ibcq;
+			event.event = IB_EVENT_CQ_ERR;
+			if (cq->ibcq.event_handler)
+				cq->ibcq.event_handler(&event, cq->ibcq.cq_context);
+		} else {
+			qpn = aeqe.event_data0;
+			qp = find_qp_by_qpn(dev, qpn);
+			if (!qp)
+				continue;
+
+			event.device = qp->ibqp.device;
+			event.element.qp = &qp->ibqp;
+			event.event = IB_EVENT_QP_FATAL;
+			if (qp->ibqp.event_handler)
+				qp->ibqp.event_handler(&event, qp->ibqp.qp_context);
+		}
+	}
+
+	notify_eq(&dev->aeq.eq);
+}
+
+int erdma_aeq_init(struct erdma_dev *dev)
+{
+	struct erdma_eq *eq = &dev->aeq.eq;
+	u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT;
+
+	eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, buf_size + ERDMA_EXTRA_BUFFER_SIZE,
+		&eq->qbuf_dma_addr, GFP_KERNEL);
+	if (!eq->qbuf)
+		return -ENOMEM;
+
+	eq->db_info = eq->qbuf + buf_size;
+
+	memset(eq->qbuf, 0, buf_size);
+	memset(eq->db_info, 0, 8);
+
+	spin_lock_init(&eq->lock);
+	atomic64_set(&eq->event_num, 0);
+	atomic64_set(&eq->notify_num, 0);
+
+	eq->depth = ERDMA_DEFAULT_EQ_DEPTH;
+	eq->db_addr = (u64 __iomem *)(dev->func_bar + ERDMA_REGS_AEQ_DB_REG);
+	eq->ci = 0;
+
+	eq->owner = 1;
+	dev->aeq.dev = dev;
+
+	dev->aeq.ready = 1;
+
+	erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_H_REG, upper_32_bits(eq->qbuf_dma_addr));
+	erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_L_REG, lower_32_bits(eq->qbuf_dma_addr));
+	erdma_reg_write32(dev, ERDMA_REGS_AEQ_DEPTH_REG, eq->depth);
+	erdma_reg_write64(dev, ERDMA_AEQ_DB_HOST_ADDR_REG, eq->qbuf_dma_addr + buf_size);
+
+	return 0;
+}
+
+void erdma_aeq_destroy(struct erdma_dev *dev)
+{
+	struct erdma_eq *eq = &dev->aeq.eq;
+	u32 buf_size  = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT;
+
+	dev->aeq.ready = 0;
+
+	dma_free_coherent(&dev->pdev->dev, buf_size + ERDMA_EXTRA_BUFFER_SIZE,
+		eq->qbuf, eq->qbuf_dma_addr);
+}
+
+
+#define MAX_POLL_CHUNK_SIZE 16
+void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb)
+{
+	int cqn;
+	struct erdma_cq *cq;
+	struct erdma_dev *dev = ceq_cb->dev;
+	u32 poll_cnt = 0;
+
+	if (!ceq_cb->ready)
+		return;
+
+	while ((cqn = erdma_poll_ceq_event(&ceq_cb->eq)) != -1) {
+		poll_cnt++;
+		if (cqn == 0)
+			continue;
+
+		cq = find_cq_by_cqn(dev, cqn);
+		if (!cq)
+			continue;
+
+		if (cq->ibcq.comp_handler)
+			cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
+
+		if (poll_cnt >= MAX_POLL_CHUNK_SIZE)
+			break;
+	}
+
+	notify_eq(&ceq_cb->eq);
+}
+
+
+static irqreturn_t erdma_intr_ceq_handler(int irq, void *data)
+{
+	struct erdma_eq_cb *ceq_cb = data;
+
+	tasklet_schedule(&ceq_cb->tasklet);
+
+	return IRQ_HANDLED;
+}
+
+void erdma_intr_ceq_task(unsigned long data)
+{
+	erdma_ceq_completion_handler((struct erdma_eq_cb *)data);
+}
+
+static int erdma_set_ceq_irq(struct erdma_dev *dev, u16 eqn)
+{
+	u32 cpu;
+	int err;
+	struct erdma_irq_info *irq_info = &dev->ceqs[eqn - 1].irq_info;
+
+	snprintf(irq_info->name, ERDMA_IRQNAME_SIZE, "erdma-ceq%u@pci:%s",
+		eqn - 1, pci_name(dev->pdev));
+	irq_info->handler = erdma_intr_ceq_handler;
+	irq_info->data = &dev->ceqs[eqn - 1];
+	irq_info->msix_vector = pci_irq_vector(dev->pdev, eqn);
+
+	tasklet_init(&dev->ceqs[eqn - 1].tasklet, erdma_intr_ceq_task,
+			(unsigned long)&dev->ceqs[eqn - 1]);
+
+	cpu = cpumask_local_spread(eqn, dev->numa_node);
+	irq_info->cpu = cpu;
+	cpumask_set_cpu(cpu, &irq_info->affinity_hint_mask);
+	dev_info(&dev->pdev->dev, "setup irq:%p vector:%d name:%s\n",
+		 irq_info,
+		 irq_info->msix_vector,
+		 irq_info->name);
+
+	err = request_irq(irq_info->msix_vector, irq_info->handler, 0,
+		irq_info->name, irq_info->data);
+	if (err) {
+		dev_err(&dev->pdev->dev, "failed to request_irq(%d)\n", err);
+		return err;
+	}
+
+	irq_set_affinity_hint(irq_info->msix_vector, &irq_info->affinity_hint_mask);
+
+	return 0;
+}
+
+
+static void erdma_free_ceq_irq(struct erdma_dev *dev, u16 eqn)
+{
+	struct erdma_irq_info *irq_info = &dev->ceqs[eqn - 1].irq_info;
+
+	irq_set_affinity_hint(irq_info->msix_vector, NULL);
+	free_irq(irq_info->msix_vector, irq_info->data);
+}
+
+static inline int
+create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq)
+{
+	int err;
+	struct erdma_cmdq_create_eq_req req;
+	dma_addr_t db_info_dma_addr;
+
+	ERDMA_CMDQ_BUILD_REQ_HDR(&req, CMDQ_SUBMOD_COMMON, CMDQ_OPCODE_CREATE_EQ);
+	req.eqn = eqn;
+	req.depth = ilog2(eq->depth);
+	req.qbuf_addr = eq->qbuf_dma_addr;
+	req.qtype = 1; /* CEQ */
+	/* Vector index is the same sa EQN. */
+	req.vector_idx = eqn;
+	db_info_dma_addr = eq->qbuf_dma_addr + (eq->depth << EQE_SHIFT);
+	req.db_dma_addr_l = lower_32_bits(db_info_dma_addr);
+	req.db_dma_addr_h = upper_32_bits(db_info_dma_addr);
+
+	err = erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req,
+		sizeof(struct erdma_cmdq_create_eq_req), NULL, NULL);
+	if (err) {
+		dev_err(&dev->pdev->dev,
+			"ERROR: err code = %d, cmd of create eq failed.\n", err);
+		return err;
+	}
+
+	return 0;
+}
+
+static int erdma_ceq_init_one(struct erdma_dev *dev, u16 eqn)
+{
+	/* CEQ indexed from 1, 0 rsvd for CMDQ-EQ. */
+	struct erdma_eq *eq = &dev->ceqs[eqn - 1].eq;
+	u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT;
+	int ret;
+
+	eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, buf_size + ERDMA_EXTRA_BUFFER_SIZE,
+		&eq->qbuf_dma_addr, GFP_KERNEL);
+	if (!eq->qbuf)
+		return -ENOMEM;
+
+	eq->db_info = eq->qbuf + ERDMA_EXTRA_BUFFER_SIZE;
+
+	memset(eq->qbuf, 0, buf_size);
+	memset(eq->db_info, 0, ERDMA_EXTRA_BUFFER_SIZE);
+
+	spin_lock_init(&eq->lock);
+	atomic64_set(&eq->event_num, 0);
+	atomic64_set(&eq->notify_num, 0);
+
+	eq->depth = ERDMA_DEFAULT_EQ_DEPTH;
+	eq->db_addr = (u64 __iomem *)(dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG + eqn * 8);
+	eq->ci = 0;
+	eq->owner = 1;
+	dev->ceqs[eqn - 1].dev = dev;
+
+	ret = create_eq_cmd(dev, eqn, eq);
+	if (ret) {
+		dev->ceqs[eqn - 1].ready = 0;
+		return ret;
+	}
+
+	dev->ceqs[eqn - 1].ready = 1;
+
+	return ret;
+}
+
+static void erdma_ceq_uninit_one(struct erdma_dev *dev, u16 eqn)
+{
+	struct erdma_eq *eq = &dev->ceqs[eqn - 1].eq;
+	u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT;
+	struct erdma_cmdq_destroy_eq_req req;
+	int err;
+
+	dev->ceqs[eqn - 1].ready = 0;
+
+	ERDMA_CMDQ_BUILD_REQ_HDR(&req, CMDQ_SUBMOD_COMMON, CMDQ_OPCODE_DESTROY_EQ);
+	req.eqn = eqn;
+	req.qtype = 1;
+	req.vector_idx = eqn;
+
+	err = erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL, NULL);
+	if (err) {
+		dev_err(&dev->pdev->dev,
+			"ERROR: err code = %d, cmd of destroy eq failed.\n", err);
+		return;
+	}
+
+	dma_free_coherent(&dev->pdev->dev, buf_size + ERDMA_EXTRA_BUFFER_SIZE,
+		eq->qbuf, eq->qbuf_dma_addr);
+}
+
+int erdma_ceqs_init(struct erdma_dev *dev)
+{
+	u32 i, j;
+	int err = 0;
+
+	for (i = 1; i < dev->irq_num; i++) {
+		err = erdma_ceq_init_one(dev, i);
+		if (err)
+			goto out_err;
+
+		err = erdma_set_ceq_irq(dev, i);
+		if (err) {
+			erdma_ceq_uninit_one(dev, i);
+			goto out_err;
+		}
+	}
+
+	return 0;
+
+out_err:
+	for (j = 1; j < i; j++) {
+		erdma_free_ceq_irq(dev, j);
+		erdma_ceq_uninit_one(dev, j);
+	}
+
+	return err;
+}
+
+void erdma_ceqs_uninit(struct erdma_dev *dev)
+{
+	u32 i;
+
+	for (i = 1; i < dev->irq_num; i++) {
+		erdma_free_ceq_irq(dev, i);
+		erdma_ceq_uninit_one(dev, i);
+	}
+}

new file mode 100644
index 000000000000..6eda8843d0d5
@@ -0,0 +1,366 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Authors: Cheng Xu <[email protected]>
+ *          Kai Shen <[email protected]>
+ * Copyright (c) 2020-2021, Alibaba Group.
+ */
+
+#ifndef __ERDMA_VERBS_H__
+#define __ERDMA_VERBS_H__
+
+#include <linux/errno.h>
+
+#include <rdma/iw_cm.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_user_verbs.h>
+
+#include "erdma.h"
+#include "erdma_cm.h"
+#include "erdma_hw.h"
+
+/* RDMA Capbility. */
+#define ERDMA_MAX_PD         (128 * 1024)
+#define ERDMA_MAX_SEND_WR    4096
+#define ERDMA_MAX_ORD        128
+#define ERDMA_MAX_IRD        128
+#define ERDMA_MAX_SGE_RD     1
+#define ERDMA_MAX_FMR        0
+#define ERDMA_MAX_SRQ        0 /* not support srq now. */
+#define ERDMA_MAX_SRQ_WR     0 /* not support srq now. */
+#define ERDMA_MAX_SRQ_SGE    0 /* not support srq now. */
+#define ERDMA_MAX_CONTEXT    (128 * 1024)
+#define ERDMA_MAX_SEND_SGE   6
+#define ERDMA_MAX_RECV_SGE   1
+#define ERDMA_MAX_INLINE     (sizeof(struct erdma_sge) * (ERDMA_MAX_SEND_SGE))
+#define ERDMA_MAX_FRMR_PA    512
+
+enum {
+	ERDMA_MMAP_IO_NC = 0,  /* no cache */
+};
+
+struct erdma_user_mmap_entry {
+	struct rdma_user_mmap_entry rdma_entry;
+	u64 address;
+	u8 mmap_flag;
+};
+
+struct erdma_ucontext {
+	struct ib_ucontext ibucontext;
+	struct erdma_dev *dev;
+
+	u32 sdb_type;
+	u32 sdb_idx;
+	u32 sdb_page_idx;
+	u32 sdb_page_off;
+	u64 sdb;
+	u64 rdb;
+	u64 cdb;
+
+	struct rdma_user_mmap_entry *sq_db_mmap_entry;
+	struct rdma_user_mmap_entry *rq_db_mmap_entry;
+	struct rdma_user_mmap_entry *cq_db_mmap_entry;
+
+	/* doorbell records */
+	struct list_head dbrecords_page_list;
+	struct mutex dbrecords_page_mutex;
+};
+
+
+struct erdma_pd {
+	struct ib_pd ibpd;
+	u32 pdn;
+};
+
+/*
+ * MemoryRegion definition.
+ */
+#define ERDMA_MAX_INLINE_MTT_ENTRIES 4
+#define MTT_SIZE(x) (x << 3) /* per mtt takes 8 Bytes. */
+#define ERDMA_MR_MAX_MTT_CNT  524288
+#define ERDMA_MTT_ENTRY_SIZE  8
+
+#define ERDMA_MR_TYPE_NORMAL  0
+#define ERDMA_MR_TYPE_FRMR    1
+#define ERDMA_MR_TYPE_DMA     2
+
+#define ERDMA_MR_INLINE_MTT   0
+#define ERDMA_MR_INDIRECT_MTT 1
+
+#define ERDMA_MR_ACC_LR BIT(0)
+#define ERDMA_MR_ACC_LW BIT(1)
+#define ERDMA_MR_ACC_RR BIT(2)
+#define ERDMA_MR_ACC_RW BIT(3)
+
+struct erdma_mem {
+	struct ib_umem *umem;
+	void *mtt_buf;
+	u32 mtt_type;
+	u32 page_size;
+	u32 page_offset;
+	u32 page_cnt;
+	u32 mtt_nents;
+
+	u64 va;
+	u64 len;
+
+	u64 mtt_entry[ERDMA_MAX_INLINE_MTT_ENTRIES];
+};
+
+struct erdma_mr {
+	struct ib_mr ibmr;
+	struct erdma_mem mem;
+	u8 type;
+	u8 access;
+	u8 valid;
+};
+
+struct erdma_user_dbrecords_page {
+	struct list_head list;
+	struct ib_umem *umem;
+	u64 va;
+	int refcnt;
+};
+
+
+struct erdma_uqp {
+	struct erdma_mem sq_mtt;
+	struct erdma_mem rq_mtt;
+
+	dma_addr_t sq_db_info_dma_addr;
+	dma_addr_t rq_db_info_dma_addr;
+
+	struct erdma_user_dbrecords_page *user_dbr_page;
+
+	u32 rq_offset;
+};
+struct erdma_kqp {
+	u16 sq_pi;
+	u16 sq_ci;
+
+	u16 rq_pi;
+	u16 rq_ci;
+
+	u64 *swr_tbl;
+	u64 *rwr_tbl;
+
+	void *hw_sq_db;
+	void *hw_rq_db;
+
+	void *sq_buf;
+	dma_addr_t sq_buf_dma_addr;
+
+	void *rq_buf;
+	dma_addr_t rq_buf_dma_addr;
+
+	void *sq_db_info;
+	void *rq_db_info;
+
+	u8 sig_all;
+};
+
+enum erdma_qp_state {
+	ERDMA_QP_STATE_IDLE      = 0,
+	ERDMA_QP_STATE_RTR       = 1,
+	ERDMA_QP_STATE_RTS       = 2,
+	ERDMA_QP_STATE_CLOSING   = 3,
+	ERDMA_QP_STATE_TERMINATE = 4,
+	ERDMA_QP_STATE_ERROR     = 5,
+	ERDMA_QP_STATE_UNDEF     = 7,
+	ERDMA_QP_STATE_COUNT     = 8
+};
+
+enum erdma_qp_flags {
+	ERDMA_BIND_ENABLED	= (1 << 0),
+	ERDMA_WRITE_ENABLED	= (1 << 1),
+	ERDMA_READ_ENABLED	= (1 << 2)
+};
+
+enum erdma_qp_attr_mask {
+	ERDMA_QP_ATTR_STATE             = (1 << 0),
+	ERDMA_QP_ATTR_ACCESS_FLAGS      = (1 << 1),
+	ERDMA_QP_ATTR_LLP_HANDLE        = (1 << 2),
+	ERDMA_QP_ATTR_ORD               = (1 << 3),
+	ERDMA_QP_ATTR_IRD               = (1 << 4),
+	ERDMA_QP_ATTR_SQ_SIZE           = (1 << 5),
+	ERDMA_QP_ATTR_RQ_SIZE           = (1 << 6),
+	ERDMA_QP_ATTR_MPA               = (1 << 7)
+};
+
+struct erdma_qp_attrs {
+	enum erdma_qp_state state;
+	u32 sq_size;
+	u32 rq_size;
+	u32 orq_size;
+	u32 irq_size;
+	u32 max_send_sge;
+	u32 max_recv_sge;
+	enum erdma_qp_flags flags;
+
+	struct socket *llp_stream_handle;
+	u32 sip;
+	u32 dip;
+	u16 sport;
+	u16 dport;
+	u16 origin_sport;
+	u32 remote_qpn;
+};
+
+struct erdma_qp {
+	struct ib_qp ibqp;
+	struct kref ref;
+	struct completion safe_free;
+	struct erdma_dev *dev;
+	struct erdma_cep *cep;
+	struct rw_semaphore state_lock;
+	bool is_kernel_qp;
+
+	union {
+		struct erdma_kqp kern_qp;
+		struct erdma_uqp user_qp;
+	};
+
+	struct erdma_cq *scq;
+	struct erdma_cq *rcq;
+
+	struct erdma_qp_attrs attrs;
+	spinlock_t lock;
+
+	u8 cc_method;
+#define ERDMA_QP_TYPE_CLIENT 0
+#define ERDMA_QP_TYPE_SERVER 1
+	u8 qp_type;
+	u8 private_data_len;
+};
+
+
+struct erdma_kcq_info {
+	struct erdma_cqe *qbuf;
+	dma_addr_t qbuf_dma_addr;
+	u32 ci;
+	u32 owner;
+	u32 cmdsn;
+	void *db;
+	spinlock_t lock;
+	void *db_info;
+};
+
+struct erdma_ucq_info {
+	struct erdma_mem qbuf_mtt;
+	struct erdma_user_dbrecords_page *user_dbr_page;
+	dma_addr_t db_info_dma_addr;
+};
+
+struct erdma_cq {
+	struct ib_cq ibcq;
+	u32 cqn;
+
+	u32 depth;
+	u32 assoc_eqn;
+	u32 is_kernel_cq;
+
+	union {
+		struct erdma_kcq_info kern_cq;
+		struct erdma_ucq_info user_cq;
+	};
+};
+
+#define QP_ID(qp) ((qp)->ibqp.qp_num)
+
+static inline struct erdma_qp *find_qp_by_qpn(struct erdma_dev *dev, int id)
+{
+	return (struct erdma_qp *)xa_load(&dev->qp_xa, id);
+}
+
+static inline struct erdma_cq *find_cq_by_cqn(struct erdma_dev *dev, int id)
+{
+	return (struct erdma_cq *)xa_load(&dev->cq_xa, id);
+}
+
+extern void erdma_qp_get(struct erdma_qp *qp);
+extern void erdma_qp_put(struct erdma_qp *qp);
+extern int erdma_modify_qp_internal(struct erdma_qp *qp, struct erdma_qp_attrs *attrs,
+				    enum erdma_qp_attr_mask mask);
+extern void erdma_qp_llp_close(struct erdma_qp *qp);
+extern void erdma_qp_cm_drop(struct erdma_qp *qp, int sched);
+
+static inline struct erdma_ucontext *to_ectx(struct ib_ucontext *ibctx)
+{
+	return container_of(ibctx, struct erdma_ucontext, ibucontext);
+}
+
+static inline struct erdma_pd *to_epd(struct ib_pd *pd)
+{
+	return container_of(pd, struct erdma_pd, ibpd);
+}
+
+static inline struct erdma_mr *to_emr(struct ib_mr *ibmr)
+{
+	return container_of(ibmr, struct erdma_mr, ibmr);
+}
+
+static inline struct erdma_qp *to_eqp(struct ib_qp *qp)
+{
+	return container_of(qp, struct erdma_qp, ibqp);
+}
+
+static inline struct erdma_cq *to_ecq(struct ib_cq *ibcq)
+{
+	return container_of(ibcq, struct erdma_cq, ibcq);
+}
+
+static inline struct erdma_user_mmap_entry *to_emmap(struct rdma_user_mmap_entry *ibmmap)
+{
+	return container_of(ibmmap, struct erdma_user_mmap_entry, rdma_entry);
+}
+
+static inline void *get_sq_entry(struct erdma_qp *qp, u16 idx)
+{
+	idx &= (qp->attrs.sq_size - 1);
+	return qp->kern_qp.sq_buf + (idx << SQEBB_SHIFT);
+}
+
+extern int erdma_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *data);
+extern void erdma_dealloc_ucontext(struct ib_ucontext *ctx);
+extern int erdma_query_device(struct ib_device *dev, struct ib_device_attr *attr,
+			      struct ib_udata *data);
+extern int erdma_get_port_immutable(struct ib_device *dev, u32 port,
+				    struct ib_port_immutable *ib_port_immutable);
+extern int erdma_create_cq(struct ib_cq *cq,
+			   const struct ib_cq_init_attr *attr,
+			   struct ib_udata *data);
+extern int erdma_query_port(struct ib_device *dev, u32 port, struct ib_port_attr *attr);
+extern int erdma_query_pkey(struct ib_device *dev, u32 port, u16 idx, u16 *pkey);
+extern int erdma_query_gid(struct ib_device *dev, u32 port, int idx, union ib_gid *gid);
+extern int erdma_alloc_pd(struct ib_pd *pd, struct ib_udata *data);
+extern int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
+extern int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
+				   struct ib_udata *data);
+extern int erdma_query_qp(struct ib_qp *qp, struct ib_qp_attr *attr, int mask,
+			struct ib_qp_init_attr *init_attr);
+extern int erdma_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr, int mask,
+			      struct ib_udata *data);
+extern int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
+extern int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+extern int erdma_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
+extern struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
+				      u64 virt, int access, struct ib_udata *udata);
+extern struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int rights);
+extern int erdma_dereg_mr(struct ib_mr *mr, struct ib_udata *data);
+extern int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma);
+extern void erdma_qp_get_ref(struct ib_qp *qp);
+extern void erdma_qp_put_ref(struct ib_qp *qp);
+extern struct ib_qp *erdma_get_ibqp(struct ib_device *dev, int id);
+extern int erdma_post_send(struct ib_qp *qp, const struct ib_send_wr *send_wr,
+			   const struct ib_send_wr **bad_send_wr);
+extern int erdma_post_recv(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,
+			   const struct ib_recv_wr **bad_recv_wr);
+extern int erdma_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
+extern struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
+				       u32 max_num_sg);
+extern int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
+			   int sg_nents, unsigned int *sg_offset);
+extern struct net_device *erdma_get_netdev(struct ib_device *device, u32 port_num);
+extern void erdma_disassociate_ucontext(struct ib_ucontext *ibcontext);
+extern void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason);
+
+#endif

new file mode 100644
index 000000000000..11bf1b26c05a
@@ -0,0 +1,201 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Authors: Cheng Xu <[email protected]>
+ *          Kai Shen <[email protected]>
+ * Copyright (c) 2020-2021, Alibaba Group.
+ */
+
+#include <rdma/ib_verbs.h>
+
+#include "erdma_hw.h"
+#include "erdma_verbs.h"
+
+static inline int erdma_cq_notempty(struct erdma_cq *cq)
+{
+	struct erdma_cqe *cqe;
+	unsigned long flags;
+	u32 hdr;
+
+	spin_lock_irqsave(&cq->kern_cq.lock, flags);
+
+	cqe = &cq->kern_cq.qbuf[cq->kern_cq.ci & (cq->depth - 1)];
+	hdr = be32_to_cpu(READ_ONCE(cqe->hdr));
+	if (FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK, hdr) != cq->kern_cq.owner) {
+		spin_unlock_irqrestore(&cq->kern_cq.lock, flags);
+		return 0;
+	}
+
+	spin_unlock_irqrestore(&cq->kern_cq.lock, flags);
+	return 1;
+}
+
+static inline void notify_cq(struct erdma_cq *cq, u8 solcitied)
+{
+	u64 db_data = FIELD_PREP(ERDMA_CQDB_EQN_MASK, cq->assoc_eqn) |
+		FIELD_PREP(ERDMA_CQDB_CQN_MASK, cq->cqn) |
+		FIELD_PREP(ERDMA_CQDB_ARM_MASK, 1) |
+		FIELD_PREP(ERDMA_CQDB_SOL_MASK, solcitied) |
+		FIELD_PREP(ERDMA_CQDB_CMDSN_MASK, cq->kern_cq.cmdsn) |
+		FIELD_PREP(ERDMA_CQDB_CI_MASK, cq->kern_cq.ci);
+
+	*(u64 *)cq->kern_cq.db_info = db_data;
+	writeq(db_data, cq->kern_cq.db);
+}
+
+int erdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
+{
+	struct erdma_cq *cq = to_ecq(ibcq);
+	int ret = 0;
+
+	notify_cq(cq, (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED);
+
+	if (flags & IB_CQ_REPORT_MISSED_EVENTS)
+		ret = erdma_cq_notempty(cq);
+
+	return ret;
+}
+
+static const struct {
+	enum erdma_opcode erdma;
+	enum ib_wc_opcode base;
+} map_cqe_opcode[ERDMA_NUM_OPCODES] = {
+	{ ERDMA_OP_WRITE, IB_WC_RDMA_WRITE },
+	{ ERDMA_OP_READ, IB_WC_RDMA_READ },
+	{ ERDMA_OP_SEND, IB_WC_SEND },
+	{ ERDMA_OP_SEND_WITH_IMM, IB_WC_SEND },
+	{ ERDMA_OP_RECEIVE, IB_WC_RECV },
+	{ ERDMA_OP_RECV_IMM, IB_WC_RECV_RDMA_WITH_IMM },
+	{ ERDMA_OP_RECV_INV, IB_WC_LOCAL_INV}, /* confirm afterwards */
+	{ ERDMA_OP_REQ_ERR, IB_WC_RECV }, /* remove afterwards */
+	{ ERDNA_OP_READ_RESPONSE, IB_WC_RECV }, /* can not appear */
+	{ ERDMA_OP_WRITE_WITH_IMM, IB_WC_RDMA_WRITE },
+	{ ERDMA_OP_RECV_ERR, IB_WC_RECV_RDMA_WITH_IMM }, /* can not appear */
+	{ ERDMA_OP_INVALIDATE, IB_WC_LOCAL_INV },
+	{ ERDMA_OP_RSP_SEND_IMM, IB_WC_RECV },
+	{ ERDMA_OP_SEND_WITH_INV, IB_WC_SEND },
+	{ ERDMA_OP_REG_MR, IB_WC_REG_MR },
+	{ ERDMA_OP_LOCAL_INV, IB_WC_LOCAL_INV },
+	{ ERDMA_OP_READ_WITH_INV, IB_WC_RDMA_READ },
+};
+
+static const struct {
+	enum erdma_wc_status erdma;
+	enum ib_wc_status base;
+	enum erdma_vendor_err vendor;
+} map_cqe_status[ERDMA_NUM_WC_STATUS] = {
+	{ ERDMA_WC_SUCCESS, IB_WC_SUCCESS, ERDMA_WC_VENDOR_NO_ERR },
+	{ ERDMA_WC_GENERAL_ERR, IB_WC_GENERAL_ERR, ERDMA_WC_VENDOR_NO_ERR },
+	{ ERDMA_WC_RECV_WQE_FORMAT_ERR, IB_WC_GENERAL_ERR, ERDMA_WC_VENDOR_INVALID_RQE },
+	{ ERDMA_WC_RECV_STAG_INVALID_ERR, IB_WC_REM_ACCESS_ERR,
+			ERDMA_WC_VENDOR_RQE_INVALID_STAG },
+	{ ERDMA_WC_RECV_ADDR_VIOLATION_ERR, IB_WC_REM_ACCESS_ERR,
+			ERDMA_WC_VENDOR_RQE_ADDR_VIOLATION },
+	{ ERDMA_WC_RECV_RIGHT_VIOLATION_ERR, IB_WC_REM_ACCESS_ERR,
+			ERDMA_WC_VENDOR_RQE_ACCESS_RIGHT_ERR },
+	{ ERDMA_WC_RECV_PDID_ERR, IB_WC_REM_ACCESS_ERR, ERDMA_WC_VENDOR_RQE_INVALID_PD },
+	{ ERDMA_WC_RECV_WARRPING_ERR, IB_WC_REM_ACCESS_ERR, ERDMA_WC_VENDOR_RQE_WRAP_ERR },
+	{ ERDMA_WC_SEND_WQE_FORMAT_ERR, IB_WC_LOC_QP_OP_ERR, ERDMA_WC_VENDOR_INVALID_SQE },
+	{ ERDMA_WC_SEND_WQE_ORD_EXCEED, IB_WC_GENERAL_ERR, ERDMA_WC_VENDOR_ZERO_ORD },
+	{ ERDMA_WC_SEND_STAG_INVALID_ERR, IB_WC_LOC_ACCESS_ERR,
+			ERDMA_WC_VENDOR_SQE_INVALID_STAG },
+	{ ERDMA_WC_SEND_ADDR_VIOLATION_ERR, IB_WC_LOC_ACCESS_ERR,
+			ERDMA_WC_VENDOR_SQE_ADDR_VIOLATION },
+	{ ERDMA_WC_SEND_RIGHT_VIOLATION_ERR, IB_WC_LOC_ACCESS_ERR,
+			ERDMA_WC_VENDOR_SQE_ACCESS_ERR },
+	{ ERDMA_WC_SEND_PDID_ERR, IB_WC_LOC_ACCESS_ERR, ERDMA_WC_VENDOR_SQE_INVALID_PD },
+	{ ERDMA_WC_SEND_WARRPING_ERR, IB_WC_LOC_ACCESS_ERR, ERDMA_WC_VENDOR_SQE_WARP_ERR },
+	{ ERDMA_WC_FLUSH_ERR, IB_WC_WR_FLUSH_ERR, ERDMA_WC_VENDOR_NO_ERR },
+	{ ERDMA_WC_RETRY_EXC_ERR, IB_WC_RETRY_EXC_ERR, ERDMA_WC_VENDOR_NO_ERR },
+};
+
+static int
+erdma_poll_one_cqe(struct erdma_cq *cq, struct erdma_cqe *cqe, struct ib_wc *wc)
+{
+	struct erdma_dev *dev = to_edev(cq->ibcq.device);
+	struct erdma_qp *qp;
+	struct erdma_kqp *kern_qp;
+	u64 *wqe_hdr;
+	u64 *id_table;
+	u32 qpn = be32_to_cpu(cqe->qpn);
+	u16 wqe_idx = be32_to_cpu(cqe->qe_idx);
+	u32 hdr = be32_to_cpu(cqe->hdr);
+	u16 depth;
+	u8 opcode, syndrome, qtype;
+
+	qp = find_qp_by_qpn(dev, qpn);
+	kern_qp = &qp->kern_qp;
+
+	qtype = FIELD_GET(ERDMA_CQE_HDR_QTYPE_MASK, hdr);
+	syndrome = FIELD_GET(ERDMA_CQE_HDR_SYNDROME_MASK, hdr);
+	opcode = FIELD_GET(ERDMA_CQE_HDR_OPCODE_MASK, hdr);
+
+	if (qtype == ERDMA_CQE_QTYPE_SQ) {
+		id_table = kern_qp->swr_tbl;
+		depth = qp->attrs.sq_size;
+		wqe_hdr = (u64 *)get_sq_entry(qp, wqe_idx);
+		kern_qp->sq_ci = wqe_idx + FIELD_GET(ERDMA_SQE_HDR_WQEBB_CNT_MASK, *wqe_hdr) + 1;
+	} else {
+		id_table = kern_qp->rwr_tbl;
+		depth = qp->attrs.rq_size;
+	}
+	wc->wr_id = id_table[wqe_idx & (depth - 1)];
+	wc->byte_len = be32_to_cpu(cqe->size);
+
+	wc->wc_flags = 0;
+
+	wc->opcode = map_cqe_opcode[opcode].base;
+	if (wc->opcode == IB_WC_RECV_RDMA_WITH_IMM) {
+		wc->ex.imm_data = be32_to_cpu(cqe->imm_data);
+		wc->wc_flags |= IB_WC_WITH_IMM;
+	}
+
+	if (syndrome >= ERDMA_NUM_WC_STATUS)
+		syndrome = ERDMA_WC_GENERAL_ERR;
+
+	wc->status = map_cqe_status[syndrome].base;
+	wc->vendor_err = map_cqe_status[syndrome].vendor;
+	wc->qp = &qp->ibqp;
+
+	return 0;
+}
+
+int erdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
+{
+	struct erdma_cq *cq = to_ecq(ibcq);
+	struct erdma_cqe *cqe;
+	unsigned long flags;
+	u32 owner;
+	u32 ci;
+	int i, ret;
+	int new = 0;
+	u32 hdr;
+
+	spin_lock_irqsave(&cq->kern_cq.lock, flags);
+
+	owner = cq->kern_cq.owner;
+	ci = cq->kern_cq.ci;
+
+	for (i = 0; i < num_entries; i++) {
+		cqe = &cq->kern_cq.qbuf[ci & (cq->depth - 1)];
+
+		hdr = be32_to_cpu(READ_ONCE(cqe->hdr));
+		if (FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK, hdr) != owner)
+			break;
+
+		/* cqbuf should be ready when we poll*/
+		dma_rmb();
+		ret = erdma_poll_one_cqe(cq, cqe, wc);
+		ci++;
+		if ((ci & (cq->depth - 1)) == 0)
+			owner = !owner;
+		if (ret)
+			continue;
+		wc++;
+		new++;
+	}
+	cq->kern_cq.owner = owner;
+	cq->kern_cq.ci = ci;
+
+	spin_unlock_irqrestore(&cq->kern_cq.lock, flags);
+	return new;
+}

624 lines not on this page · raw message

1,477 lines not on this page · raw message

new file mode 100644
index 000000000000..36d4f353d5c6
@@ -0,0 +1,1585 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Authors: Cheng Xu <[email protected]>
+ *          Kai Shen <[email protected]>
+ * Copyright (c) 2020-2021, Alibaba Group.
+ *
+ * Authors: Bernard Metzler <[email protected]>
+ *          Fredy Neeser <[email protected]>
+ * Copyright (c) 2008-2016, IBM Corporation
+ */
+
+#include <linux/errno.h>
+#include <linux/inetdevice.h>
+#include <linux/net.h>
+#include <linux/inetdevice.h>
+#include <net/addrconf.h>
+#include <linux/tcp.h>
+#include <linux/types.h>
+#include <linux/workqueue.h>
+#include <net/sock.h>
+
+#include <rdma/iw_cm.h>
+#include <rdma/ib_smi.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_verbs.h>
+
+#include "erdma.h"
+#include "erdma_cm.h"
+#include "erdma_verbs.h"
+
+static bool mpa_crc_strict = 1;
+module_param(mpa_crc_strict, bool, 0644);
+static bool mpa_crc_required;
+module_param(mpa_crc_required, bool, 0644);
+
+MODULE_PARM_DESC(mpa_crc_required, "MPA CRC required");
+MODULE_PARM_DESC(mpa_crc_strict, "MPA CRC off enforced");
+
+static void erdma_cm_llp_state_change(struct sock *sk);
+static void erdma_cm_llp_data_ready(struct sock *sk);
+static void erdma_cm_llp_error_report(struct sock *sk);
+
+static void erdma_sk_assign_cm_upcalls(struct sock *sk)
+{
+	write_lock_bh(&sk->sk_callback_lock);
+	sk->sk_state_change = erdma_cm_llp_state_change;
+	sk->sk_data_ready = erdma_cm_llp_data_ready;
+	sk->sk_error_report = erdma_cm_llp_error_report;
+	write_unlock_bh(&sk->sk_callback_lock);
+}
+
+static void erdma_sk_save_upcalls(struct sock *sk)
+{
+	struct erdma_cep *cep = sk_to_cep(sk);
+
+	WARN_ON(!cep);
+
+	write_lock_bh(&sk->sk_callback_lock);
+	cep->sk_state_change = sk->sk_state_change;
+	cep->sk_data_ready = sk->sk_data_ready;
+	cep->sk_error_report = sk->sk_error_report;
+	write_unlock_bh(&sk->sk_callback_lock);
+}
+
+static void erdma_sk_restore_upcalls(struct sock *sk, struct erdma_cep *cep)
+{
+	sk->sk_state_change = cep->sk_state_change;
+	sk->sk_data_ready = cep->sk_data_ready;
+	sk->sk_error_report = cep->sk_error_report;
+	sk->sk_user_data = NULL;
+}
+
+static void erdma_socket_disassoc(struct socket *s)
+{
+	struct sock	*sk = s->sk;
+	struct erdma_cep	*cep;
+
+	if (sk) {
+		write_lock_bh(&sk->sk_callback_lock);
+		cep = sk_to_cep(sk);
+		if (cep) {
+			erdma_sk_restore_upcalls(sk, cep);
+			erdma_cep_put(cep);
+		} else
+			pr_warn("cannot restore sk callbacks: no ep\n");
+		write_unlock_bh(&sk->sk_callback_lock);
+	} else
+		pr_warn("cannot restore sk callbacks: no sk\n");
+}
+
+static inline int kernel_peername(struct socket *s, struct sockaddr_in *addr)
+{
+	return s->ops->getname(s, (struct sockaddr *)addr, 1);
+}
+
+static inline int kernel_localname(struct socket *s, struct sockaddr_in *addr)
+{
+	return s->ops->getname(s, (struct sockaddr *)addr, 0);
+}
+
+static void erdma_cep_socket_assoc(struct erdma_cep *cep, struct socket *s)
+{
+	cep->llp.sock = s;
+	erdma_cep_get(cep);
+	s->sk->sk_user_data = cep;
+
+	erdma_sk_save_upcalls(s->sk);
+	erdma_sk_assign_cm_upcalls(s->sk);
+}
+
+
+static struct erdma_cep *erdma_cep_alloc(struct erdma_dev  *dev)
+{
+	struct erdma_cep *cep = kzalloc(sizeof(*cep), GFP_KERNEL);
+
+	if (cep) {
+		unsigned long flags;
+
+		INIT_LIST_HEAD(&cep->listenq);
+		INIT_LIST_HEAD(&cep->devq);
+		INIT_LIST_HEAD(&cep->work_freelist);
+
+		kref_init(&cep->ref);
+		cep->state = ERDMA_EPSTATE_IDLE;
+		init_waitqueue_head(&cep->waitq);
+		spin_lock_init(&cep->lock);
+		cep->dev = dev;
+
+		spin_lock_irqsave(&dev->lock, flags);
+		list_add_tail(&cep->devq, &dev->cep_list);
+		spin_unlock_irqrestore(&dev->lock, flags);
+		atomic_inc(&dev->num_cep);
+	}
+	return cep;
+}
+
+static void erdma_cm_free_work(struct erdma_cep *cep)
+{
+	struct list_head	*w, *tmp;
+	struct erdma_cm_work	*work;
+
+	list_for_each_safe(w, tmp, &cep->work_freelist) {
+		work = list_entry(w, struct erdma_cm_work, list);
+		list_del(&work->list);
+		kfree(work);
+	}
+}
+
+static void erdma_cancel_mpatimer(struct erdma_cep *cep)
+{
+	spin_lock_bh(&cep->lock);
+	if (cep->mpa_timer) {
+		if (cancel_delayed_work(&cep->mpa_timer->work)) {
+			erdma_cep_put(cep);
+			kfree(cep->mpa_timer); /* not needed again */
+		}
+		cep->mpa_timer = NULL;
+	}
+	spin_unlock_bh(&cep->lock);
+}
+
+static void erdma_put_work(struct erdma_cm_work *work)
+{
+	INIT_LIST_HEAD(&work->list);
+	spin_lock_bh(&work->cep->lock);
+	list_add(&work->list, &work->cep->work_freelist);
+	spin_unlock_bh(&work->cep->lock);
+}
+
+static void erdma_cep_set_inuse(struct erdma_cep *cep)
+{
+	unsigned long flags;
+	int ret;
+retry:
+	spin_lock_irqsave(&cep->lock, flags);
+
+	if (cep->in_use) {
+		spin_unlock_irqrestore(&cep->lock, flags);
+		ret = wait_event_interruptible(cep->waitq, !cep->in_use);
+		if (signal_pending(current))
+			flush_signals(current);
+		goto retry;
+	} else {
+		cep->in_use = 1;
+		spin_unlock_irqrestore(&cep->lock, flags);
+	}
+}
+
+static void erdma_cep_set_free(struct erdma_cep *cep)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&cep->lock, flags);
+	cep->in_use = 0;
+	spin_unlock_irqrestore(&cep->lock, flags);
+
+	wake_up(&cep->waitq);
+}
+
+
+static void __erdma_cep_dealloc(struct kref *ref)
+{
+	struct erdma_cep *cep = container_of(ref, struct erdma_cep, ref);
+	struct erdma_dev *dev = cep->dev;
+	unsigned long flags;
+
+	WARN_ON(cep->listen_cep);
+
+	/* kfree(NULL) is save */
+	if (cep->private_storage != NULL)
+		kfree(cep->private_storage);
+	if (cep->private_storage != NULL)
+		kfree(cep->mpa.pdata);
+	spin_lock_bh(&cep->lock);
+	if (!list_empty(&cep->work_freelist))
+		erdma_cm_free_work(cep);
+	spin_unlock_bh(&cep->lock);
+
+	spin_lock_irqsave(&dev->lock, flags);
+	list_del(&cep->devq);
+	spin_unlock_irqrestore(&dev->lock, flags);
+	atomic_dec(&dev->num_cep);
+	kfree(cep);
+}
+
+static struct erdma_cm_work *erdma_get_work(struct erdma_cep *cep)
+{
+	struct erdma_cm_work    *work = NULL;
+	unsigned long           flags;
+
+	spin_lock_irqsave(&cep->lock, flags);
+	if (!list_empty(&cep->work_freelist)) {
+		work = list_entry(cep->work_freelist.next, struct erdma_cm_work,
+				  list);
+		list_del_init(&work->list);
+	}
+	spin_unlock_irqrestore(&cep->lock, flags);
+	return work;
+}
+
+static int erdma_cm_alloc_work(struct erdma_cep *cep, int num)
+{
+	struct erdma_cm_work        *work;
+
+	if (!list_empty(&cep->work_freelist)) {
+		pr_err("ERROR: Not init work_freelist.\n");
+		return -ENOMEM;
+	}
+
+	while (num--) {
+		work = kmalloc(sizeof(*work), GFP_KERNEL);
+		if (!work) {
+			if (!(list_empty(&cep->work_freelist)))
+				erdma_cm_free_work(cep);
+			return -ENOMEM;
+		}
+		work->cep = cep;
+		INIT_LIST_HEAD(&work->list);
+		list_add(&work->list, &cep->work_freelist);
+	}
+	return 0;
+}
+
+/*
+ * erdma_cm_upcall()
+ *
+ * Upcall to IWCM to inform about async connection events
+ */
+static int erdma_cm_upcall(struct erdma_cep *cep, enum iw_cm_event_type reason,
+			   int status)
+{
+	struct iw_cm_event event;
+	struct iw_cm_id *cm_id;
+
+	memset(&event, 0, sizeof(event));
+	event.status = status;
+	event.event = reason;
+
+	if (reason == IW_CM_EVENT_CONNECT_REQUEST ||
+	    reason == IW_CM_EVENT_CONNECT_REPLY) {
+		u16 pd_len = be16_to_cpu(cep->mpa.hdr.params.pd_len);
+
+		if (pd_len) {
+			event.private_data_len = pd_len;
+			event.private_data = cep->mpa.pdata;
+			if (cep->mpa.pdata == NULL)
+				event.private_data_len = 0;
+		}
+
+		to_sockaddr_in(event.local_addr) = cep->llp.laddr;
+		to_sockaddr_in(event.remote_addr) = cep->llp.raddr;
+	}
+	if (reason == IW_CM_EVENT_CONNECT_REQUEST) {
+		event.ird = cep->dev->attrs.max_ird;
+		event.ord = cep->dev->attrs.max_ord;
+		event.provider_data = cep;
+		cm_id = cep->listen_cep->cm_id;
+	} else
+		cm_id = cep->cm_id;
+
+	if (!cep->is_connecting && reason == IW_CM_EVENT_CONNECT_REPLY)
+		return 0;
+
+	cep->is_connecting = false;
+
+	return cm_id->event_handler(cm_id, &event);
+}
+
+/*
+ * erdma_qp_cm_drop()
+ *
+ * Drops established LLP connection if present and not already
+ * scheduled for dropping. Called from user context, SQ workqueue
+ * or receive IRQ. Caller signals if socket can be immediately
+ * closed (basically, if not in IRQ).
+ */
+void erdma_qp_cm_drop(struct erdma_qp *qp, int schedule)
+{
+	struct erdma_cep *cep = qp->cep;
+
+	if (!qp->cep)
+		return;
+
+	if (schedule)
+		erdma_cm_queue_work(cep, ERDMA_CM_WORK_CLOSE_LLP);
+	else {
+		erdma_cep_set_inuse(cep);
+
+		if (cep->state == ERDMA_EPSTATE_CLOSED)
+			goto out;
+
+		if (cep->cm_id) {
+			switch (cep->state) {
+
+			case ERDMA_EPSTATE_AWAIT_MPAREP:
+				erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY,
+					      -EINVAL);
+				break;
+
+			case ERDMA_EPSTATE_RDMA_MODE:
+				erdma_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0);
+
+				break;
+
+			case ERDMA_EPSTATE_IDLE:
+			case ERDMA_EPSTATE_LISTENING:
+			case ERDMA_EPSTATE_CONNECTING:
+			case ERDMA_EPSTATE_AWAIT_MPAREQ:
+			case ERDMA_EPSTATE_RECVD_MPAREQ:
+			case ERDMA_EPSTATE_CLOSED:
+			default:
+
+				break;
+			}
+			cep->cm_id->rem_ref(cep->cm_id);
+			cep->cm_id = NULL;
+			erdma_cep_put(cep);
+		}
+		cep->state = ERDMA_EPSTATE_CLOSED;
+
+		if (cep->llp.sock) {
+			erdma_socket_disassoc(cep->llp.sock);
+			sock_release(cep->llp.sock);
+			cep->llp.sock = NULL;
+		}
+		if (cep->qp) {
+			WARN_ON(qp != cep->qp);
+			cep->qp = NULL;
+			erdma_qp_put(qp);
+		}
+out:
+		erdma_cep_set_free(cep);
+	}
+}
+
+
+void erdma_cep_put(struct erdma_cep *cep)
+{
+	WARN_ON(kref_read(&cep->ref) < 1);
+	kref_put(&cep->ref, __erdma_cep_dealloc);
+}
+
+void erdma_cep_get(struct erdma_cep *cep)
+{
+	kref_get(&cep->ref);
+}
+
+static inline int ksock_recv(struct socket *sock, char *buf, size_t size,
+			     int flags)
+{
+	struct kvec iov = {buf, size};
+	struct msghdr msg = {.msg_name = NULL, .msg_flags = flags};
+
+	return kernel_recvmsg(sock, &msg, &iov, 1, size, flags);
+}
+
+static inline void __mpa_rr_set_cc(u16 *bits, u16 cc)
+{
+	*bits = (*bits & ~MPA_RR_DESIRED_CC)
+		| (cc & MPA_RR_DESIRED_CC);
+}
+
+static inline u8 __mpa_rr_cc(u16 mpa_rr_bits)
+{
+	u16 rev = (mpa_rr_bits & MPA_RR_DESIRED_CC);
+
+	return (u8)rev;
+}
+
+static inline void __mpa_rr_set_revision(u16 *bits, u8 rev)
+{
+	*bits = (*bits & ~MPA_RR_MASK_REVISION)
+		| (cpu_to_be16(rev) & MPA_RR_MASK_REVISION);
+}
+
+static inline u8 __mpa_rr_revision(u16 mpa_rr_bits)
+{
+	u16 rev = mpa_rr_bits & MPA_RR_MASK_REVISION;
+
+	return (u8)be16_to_cpu(rev);
+}
+
+/*
+ * Expects params->pd_len in host byte order
+ */
+static int erdma_send_mpareqrep(struct erdma_cep *cep, const void *pdata,
+			      u8 pd_len)
+{
+	struct socket	*s = cep->llp.sock;
+	struct mpa_rr	*rr = &cep->mpa.hdr;
+	struct kvec	iov[2];
+	struct msghdr	msg;
+	int		ret;
+
+	memset(&msg, 0, sizeof(msg));
+
+	rr->params.pd_len = cpu_to_be16(pd_len);
+
+	iov[0].iov_base = rr;
+	iov[0].iov_len = sizeof(*rr);
+
+	if (pd_len) {
+		iov[1].iov_base = (char *)pdata;
+		iov[1].iov_len = pd_len;
+
+		ret =  kernel_sendmsg(s, &msg, iov, 2, pd_len + sizeof(*rr));
+	} else
+		ret =  kernel_sendmsg(s, &msg, iov, 1, sizeof(*rr));
+
+	return ret < 0 ? ret : 0;
+}
+
+/*
+ * Receive MPA Request/Reply header.
+ *
+ * Returns 0 if complete MPA Request/Reply haeder including
+ * eventual private data was received. Returns -EAGAIN if
+ * header was partially received or negative error code otherwise.
+ *
+ * Context: May be called in process context only
+ */
+static int erdma_recv_mpa_rr(struct erdma_cep *cep)
+{
+	struct mpa_rr	*hdr = &cep->mpa.hdr;
+	struct socket	*s = cep->llp.sock;
+	u16		pd_len;
+	int		rcvd, to_rcv;
+
+	if (cep->mpa.bytes_rcvd < sizeof(struct mpa_rr)) {
+
+		rcvd = ksock_recv(s, (char *)hdr + cep->mpa.bytes_rcvd,
+				  sizeof(struct mpa_rr) -
+				  cep->mpa.bytes_rcvd, MSG_DONTWAIT);
+		/* we use DONTWAIT mode, so EAGAIN may appear. */
+		if (rcvd == -EAGAIN)
+			return -EAGAIN;
+
+		if (rcvd <= 0)
+			return -ECONNABORTED;
+
+		cep->mpa.bytes_rcvd += rcvd;
+
+		if (cep->mpa.bytes_rcvd < sizeof(struct mpa_rr))
+			return -EAGAIN;
+
+		if (be16_to_cpu(hdr->params.pd_len) > MPA_MAX_PRIVDATA)
+			return -EPROTO;
+	}
+	pd_len = be16_to_cpu(hdr->params.pd_len);
+
+	/*
+	 * At least the MPA Request/Reply header (frame not including
+	 * private data) has been received.
+	 * Receive (or continue receiving) any private data.
+	 */
+	to_rcv = pd_len - (cep->mpa.bytes_rcvd - sizeof(struct mpa_rr));
+
+	if (!to_rcv) {
+		/*
+		 * We must have hdr->params.pd_len == 0 and thus received a
+		 * complete MPA Request/Reply frame.
+		 * Check against peer protocol violation.
+		 */
+		u32 word;
+
+		rcvd = ksock_recv(s, (char *)&word, sizeof(word), MSG_DONTWAIT);
+		if (rcvd == -EAGAIN)
+			return 0;
+
+		if (rcvd == 0)
+			return -EPIPE;
+
+		if (rcvd < 0)
+			return rcvd;
+		return -EPROTO;
+	}
+
+	/*
+	 * At this point, we must have hdr->params.pd_len != 0.
+	 * A private data buffer gets allocated if hdr->params.pd_len != 0.
+	 */
+	if (!cep->mpa.pdata) {
+		cep->mpa.pdata = kmalloc(pd_len + 4, GFP_KERNEL);
+		if (!cep->mpa.pdata)
+			return -ENOMEM;
+	}
+	rcvd = ksock_recv(s, cep->mpa.pdata + cep->mpa.bytes_rcvd
+			  - sizeof(struct mpa_rr), to_rcv + 4, MSG_DONTWAIT);
+
+	if (rcvd < 0)
+		return rcvd;
+
+	if (rcvd > to_rcv)
+		return -EPROTO;
+
+	cep->mpa.bytes_rcvd += rcvd;
+
+	if (to_rcv == rcvd)
+		return 0;
+
+	return -EAGAIN;
+}
+
+
+/*
+ * erdma_proc_mpareq()
+ *
+ * Read MPA Request from socket and signal new connection to IWCM
+ * if success. Caller must hold lock on corresponding listening CEP.
+ */
+static int erdma_proc_mpareq(struct erdma_cep *cep)
+{
+	struct mpa_rr      *req;
+	int                ret;
+
+	ret = erdma_recv_mpa_rr(cep);
+	if (ret)
+		goto out;
+
+	req = &cep->mpa.hdr;
+
+	if (__mpa_rr_revision(req->params.bits) > MPA_REVISION_1) {
+		/* allow for 0 and 1 only */
+		ret = -EPROTO;
+		goto out;
+	}
+
+	if (memcmp(req->key, MPA_KEY_REQ, 12)) {
+		ret = -EPROTO;
+		goto out;
+	}
+
+	cep->mpa.remote_qpn = *(u32 *)&req->key[12];
+	/*
+	 * Prepare for sending MPA reply
+	 */
+	memcpy(req->key, MPA_KEY_REP, 12);
+
+	if (req->params.bits & MPA_RR_FLAG_MARKERS ||
+	    (req->params.bits & MPA_RR_FLAG_CRC &&
+	    !mpa_crc_required && mpa_crc_strict)) {
+		req->params.bits &= ~MPA_RR_FLAG_MARKERS;
+		req->params.bits |= MPA_RR_FLAG_REJECT; /* reject */
+
+		if (!mpa_crc_required && mpa_crc_strict)
+			req->params.bits &= ~MPA_RR_FLAG_CRC;
+
+		kfree(cep->mpa.pdata);
+		cep->mpa.pdata = NULL;
+
+		(void)erdma_send_mpareqrep(cep, NULL, 0);
+		ret = -EOPNOTSUPP;
+		goto out;
+	}
+	/*
+	 * Enable CRC if requested by module initialization
+	 */
+	if (!(req->params.bits & MPA_RR_FLAG_CRC) && mpa_crc_required)
+		req->params.bits |= MPA_RR_FLAG_CRC;
+
+	cep->state = ERDMA_EPSTATE_RECVD_MPAREQ;
+
+	/* Keep reference until IWCM accepts/rejects */
+	erdma_cep_get(cep);
+	ret = erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REQUEST, 0);
+	if (ret)
+		erdma_cep_put(cep);
+out:
+	return ret;
+}
+
+static int erdma_proc_mpareply(struct erdma_cep *cep)
+{
+	struct erdma_qp_attrs qp_attrs;
+	struct erdma_qp *qp = cep->qp;
+	struct mpa_rr *rep;
+	int ret;
+
+	ret = erdma_recv_mpa_rr(cep);
+	if (ret != -EAGAIN)
+		erdma_cancel_mpatimer(cep);
+	if (ret)
+		goto out_err;
+
+	rep = &cep->mpa.hdr;
+
+	if (__mpa_rr_revision(rep->params.bits) > MPA_REVISION_1) {
+		/* allow for 0 and 1 only */
+		ret = -EPROTO;
+		goto out_err;
+	}
+	if (memcmp(rep->key, MPA_KEY_REP, 12)) {
+		ret = -EPROTO;
+		goto out_err;
+	}
+
+	cep->mpa.remote_qpn = *(u32 *)&rep->key[12];
+
+	if (rep->params.bits & MPA_RR_FLAG_REJECT) {
+		(void)erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -ECONNRESET);
+		ret = -ECONNRESET;
+		goto out;
+	}
+
+	if ((rep->params.bits & MPA_RR_FLAG_MARKERS)
+		|| (mpa_crc_required && !(rep->params.bits & MPA_RR_FLAG_CRC))
+		|| (mpa_crc_strict && !mpa_crc_required
+			&& (rep->params.bits & MPA_RR_FLAG_CRC))) {
+
+		(void)erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY,
+			-ECONNREFUSED);
+		ret = -EINVAL;
+		goto out;
+	}
+	memset(&qp_attrs, 0, sizeof(qp_attrs));
+	qp_attrs.irq_size = cep->ird;
+	qp_attrs.orq_size = cep->ord;
+	qp_attrs.llp_stream_handle = cep->llp.sock;
+	qp_attrs.state = ERDMA_QP_STATE_RTS;
+
+	down_write(&qp->state_lock);
+	if (qp->attrs.state > ERDMA_QP_STATE_RTS) {
+		ret = -EINVAL;
+		up_write(&qp->state_lock);
+		goto out_err;
+	}
+
+	qp->qp_type = ERDMA_QP_TYPE_CLIENT;
+	qp->cc_method = __mpa_rr_cc(rep->params.bits) == qp->dev->cc_method ?
+			qp->dev->cc_method : COMPROMISE_CC;
+	ret = erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE |
+						      ERDMA_QP_ATTR_LLP_HANDLE |
+						      ERDMA_QP_ATTR_MPA);
+
+	up_write(&qp->state_lock);
+
+	if (!ret) {
+		ret = erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 0);
+		if (!ret)
+			cep->state = ERDMA_EPSTATE_RDMA_MODE;
+
+		goto out;
+	}
+
+out_err:
+	(void)erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -EINVAL);
+out:
+	return ret;
+}
+
+/*
+ * erdma_accept_newconn - accept an incoming pending connection
+ *
+ */
+static void erdma_accept_newconn(struct erdma_cep *cep)
+{
+	struct socket    *s       = cep->llp.sock;
+	struct socket    *new_s   = NULL;
+	struct erdma_cep *new_cep = NULL;
+	int              ret       = 0; /* debug only. should disappear */
+
+	if (cep->state != ERDMA_EPSTATE_LISTENING)
+		goto error;
+
+	new_cep = erdma_cep_alloc(cep->dev);
+	if (!new_cep)
+		goto error;
+
+	if (erdma_cm_alloc_work(new_cep, 6) != 0)
+		goto error;
+
+	/*
+	 * Copy saved socket callbacks from listening CEP
+	 * and assign new socket with new CEP
+	 */
+	new_cep->sk_state_change = cep->sk_state_change;
+	new_cep->sk_data_ready   = cep->sk_data_ready;
+	new_cep->sk_error_report = cep->sk_error_report;
+
+	ret = kernel_accept(s, &new_s, O_NONBLOCK);
+	if (ret != 0)
+		goto error;
+
+	new_cep->llp.sock = new_s;
+	new_s->sk->sk_user_data = new_cep;
+	erdma_cep_get(new_cep);
+
+	tcp_sock_set_nodelay(new_s->sk);
+
+	ret = kernel_peername(new_s, &new_cep->llp.raddr);
+	if (ret < 0)
+		goto error;
+
+	ret = kernel_localname(new_s, &new_cep->llp.laddr);
+	if (ret < 0)
+		goto error;
+
+	new_cep->state = ERDMA_EPSTATE_AWAIT_MPAREQ;
+
+	ret = erdma_cm_queue_work(new_cep, ERDMA_CM_WORK_MPATIMEOUT);
+	if (ret)
+		goto error;
+	/*
+	 * See erdma_proc_mpareq() etc. for the use of new_cep->listen_cep.
+	 */
+	new_cep->listen_cep = cep;
+	erdma_cep_get(cep);
+
+	if (atomic_read(&new_s->sk->sk_rmem_alloc)) {
+		/*
+		 * MPA REQ already queued
+		 */
+		erdma_cep_set_inuse(new_cep);
+		ret = erdma_proc_mpareq(new_cep);
+		erdma_cep_set_free(new_cep);
+
+		if (ret != -EAGAIN) {
+			erdma_cep_put(cep);
+			new_cep->listen_cep = NULL;
+			if (ret)
+				goto error;
+		}
+	}
+	return;
+
+error:
+	if (new_cep) {
+		new_cep->state = ERDMA_EPSTATE_CLOSED;
+		erdma_cancel_mpatimer(new_cep);
+
+		erdma_cep_put(new_cep);
+		new_cep->llp.sock = NULL;
+	}
+
+	if (new_s) {
+		erdma_socket_disassoc(new_s);
+		sock_release(new_s);
+	}
+}
+
+static int erdma_newconn_connected(struct erdma_cep *cep)
+{
+	struct socket    *s       = cep->llp.sock;
+	int              ret;
+	int              qpn;
+
+	ret = kernel_peername(s, &cep->llp.raddr);
+	if (ret < 0)
+		goto error;
+
+	ret = kernel_localname(s, &cep->llp.laddr);
+	if (ret < 0)
+		goto error;
+
+	cep->mpa.hdr.params.bits = 0;
+	__mpa_rr_set_revision(&cep->mpa.hdr.params.bits, MPA_REVISION_1);
+	__mpa_rr_set_cc(&cep->mpa.hdr.params.bits, cep->dev->cc_method);
+
+	if (mpa_crc_required)
+		cep->mpa.hdr.params.bits |= MPA_RR_FLAG_CRC;
+
+	qpn = QP_ID(cep->qp);
+	memcpy(cep->mpa.hdr.key, MPA_KEY_REQ, 12);
+	memcpy(&cep->mpa.hdr.key[12], &qpn, 4);
+
+	ret = erdma_send_mpareqrep(cep, cep->private_storage, cep->pd_len);
+
+	cep->mpa.hdr.params.pd_len = 0;
+
+	if (ret >= 0) {
+		cep->state = ERDMA_EPSTATE_AWAIT_MPAREP;
+		ret = erdma_cm_queue_work(cep, ERDMA_CM_WORK_MPATIMEOUT);
+		if (!ret)
+			return 0;
+	}
+
+error:
+	return ret;
+}
+
+static void erdma_cm_work_handler(struct work_struct *w)
+{
+	struct erdma_cm_work *work;
+	struct erdma_cep     *cep;
+	int                  release_cep = 0, ret = 0;
+
+	work = container_of(w, struct erdma_cm_work, work.work);
+	cep = work->cep;
+
+	erdma_cep_set_inuse(cep);
+
+	switch (work->type) {
+	case ERDMA_CM_WORK_CONNECTED:
+		erdma_cancel_mpatimer(cep);
+		if (cep->state == ERDMA_EPSTATE_CONNECTING) {
+			ret = erdma_newconn_connected(cep);
+			if (ret) {
+				erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -EIO);
+				release_cep = 1;
+			}
+		}
+
+		break;
+	case ERDMA_CM_WORK_CONNECTTIMEOUT:
+		if (cep->state == ERDMA_EPSTATE_CONNECTING) {
+			cep->mpa_timer = NULL;
+			erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY,
+					-ETIMEDOUT);
+			release_cep = 1;
+		}
+		break;
+	case ERDMA_CM_WORK_ACCEPT:
+
+		erdma_accept_newconn(cep);
+		break;
+
+	case ERDMA_CM_WORK_READ_MPAHDR:
+		switch (cep->state) {
+		case ERDMA_EPSTATE_AWAIT_MPAREQ:
+			if (cep->listen_cep) {
+				erdma_cep_set_inuse(cep->listen_cep);
+
+				if (cep->listen_cep->state ==
+				    ERDMA_EPSTATE_LISTENING)
+					ret = erdma_proc_mpareq(cep);
+				else
+					ret = -EFAULT;
+
+				erdma_cep_set_free(cep->listen_cep);
+
+				if (ret != -EAGAIN) {
+					erdma_cep_put(cep->listen_cep);
+					cep->listen_cep = NULL;
+					if (ret)
+						erdma_cep_put(cep);
+				}
+			}
+			break;
+
+		case ERDMA_EPSTATE_AWAIT_MPAREP:
+			ret = erdma_proc_mpareply(cep);
+			break;
+		default:
+			break;
+		}
+		if (ret && ret != -EAGAIN)
+			release_cep = 1;
+		break;
+	case ERDMA_CM_WORK_CLOSE_LLP:
+		if (cep->cm_id)
+			erdma_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0);
+		release_cep = 1;
+		break;
+	case ERDMA_CM_WORK_PEER_CLOSE:
+		if (cep->cm_id) {
+			switch (cep->state) {
+			case ERDMA_EPSTATE_CONNECTING:
+			case ERDMA_EPSTATE_AWAIT_MPAREP:
+				/*
+				 * MPA reply not received, but connection drop
+				 */
+				erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY,
+					      -ECONNRESET);
+				break;
+			case ERDMA_EPSTATE_RDMA_MODE:
+				/*
+				 * NOTE: IW_CM_EVENT_DISCONNECT is given just
+				 *       to transition IWCM into CLOSING.
+				 *       FIXME: is that needed?
+				 */
+				erdma_cm_upcall(cep, IW_CM_EVENT_DISCONNECT, 0);
+				erdma_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0);
+				break;
+			default:
+				break;
+			}
+		} else {
+			switch (cep->state) {
+			case ERDMA_EPSTATE_RECVD_MPAREQ:
+				break;
+			case ERDMA_EPSTATE_AWAIT_MPAREQ:
+				/*
+				 * Socket close before MPA request received.
+				 */
+				if (cep->listen_cep) {
+					erdma_cep_put(cep->listen_cep);
+					cep->listen_cep = NULL;
+				}
+				break;
+			default:
+				break;
+			}
+		}
+		release_cep = 1;
+		break;
+	case ERDMA_CM_WORK_MPATIMEOUT:
+		cep->mpa_timer = NULL;
+		if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) {
+			/*
+			 * MPA request timed out:
+			 * Hide any partially received private data and signal
+			 * timeout
+			 */
+			cep->mpa.hdr.params.pd_len = 0;
+
+			if (cep->cm_id)
+				erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -ETIMEDOUT);
+			release_cep = 1;
+		} else if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ) {
+			/*
+			 * No MPA request received after peer TCP stream setup.
+			 */
+			if (cep->listen_cep) {
+				erdma_cep_put(cep->listen_cep);
+				cep->listen_cep = NULL;
+			}
+
+			erdma_cep_put(cep);
+			release_cep = 1;
+		}
+		break;
+	default:
+		pr_err("ERROR: work task type:%u.\n", work->type);
+		break;
+	}
+
+	if (release_cep) {
+		erdma_cancel_mpatimer(cep);
+		cep->state = ERDMA_EPSTATE_CLOSED;
+		if (cep->qp) {
+			struct erdma_qp *qp = cep->qp;
+			/*
+			 * Serialize a potential race with application
+			 * closing the QP and calling erdma_qp_cm_drop()
+			 */
+			erdma_qp_get(qp);
+			erdma_cep_set_free(cep);
+
+			erdma_qp_llp_close(qp);
+			erdma_qp_put(qp);
+
+			erdma_cep_set_inuse(cep);
+			cep->qp = NULL;
+			erdma_qp_put(qp);
+		}
+		if (cep->llp.sock) {
+			erdma_socket_disassoc(cep->llp.sock);
+			sock_release(cep->llp.sock);
+			cep->llp.sock = NULL;
+		}
+
+		if (cep->cm_id) {
+			cep->cm_id->rem_ref(cep->cm_id);
+			cep->cm_id = NULL;
+			if (cep->state != ERDMA_EPSTATE_LISTENING)
+				erdma_cep_put(cep);
+		}
+	}
+	erdma_cep_set_free(cep);
+	erdma_put_work(work);
+	erdma_cep_put(cep);
+}
+
+static struct workqueue_struct *erdma_cm_wq;
+
+int erdma_cm_queue_work(struct erdma_cep *cep, enum erdma_work_type type)
+{
+	struct erdma_cm_work *work = erdma_get_work(cep);
+	unsigned long delay = 0;
+
+	if (!work)
+		return -ENOMEM;
+
+	work->type = type;
+	work->cep = cep;
+
+	erdma_cep_get(cep);
+
+	INIT_DELAYED_WORK(&work->work, erdma_cm_work_handler);
+
+	if (type == ERDMA_CM_WORK_MPATIMEOUT) {
+		cep->mpa_timer = work;
+
+		if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREP)
+			delay = MPAREQ_TIMEOUT;
+		else
+			delay = MPAREP_TIMEOUT;
+	} else if (type == ERDMA_CM_WORK_CONNECTTIMEOUT) {
+		cep->mpa_timer = work;
+
+		delay = CONNECT_TIMEOUT;
+	}
+
+	queue_delayed_work(erdma_cm_wq, &work->work, delay);
+
+	return 0;
+}
+
+static void erdma_cm_llp_data_ready(struct sock *sk)
+{
+	struct erdma_cep *cep;
+
+	read_lock(&sk->sk_callback_lock);
+
+	cep = sk_to_cep(sk);
+	if (!cep)
+		goto out;
+
+	switch (cep->state) {
+	case ERDMA_EPSTATE_RDMA_MODE:
+	case ERDMA_EPSTATE_LISTENING:
+		break;
+	case ERDMA_EPSTATE_AWAIT_MPAREQ:
+	case ERDMA_EPSTATE_AWAIT_MPAREP:
+		erdma_cm_queue_work(cep, ERDMA_CM_WORK_READ_MPAHDR);
+		break;
+	default:
+		break;
+	}
+out:
+	read_unlock(&sk->sk_callback_lock);
+}
+
+static void erdma_cm_llp_error_report(struct sock *sk)
+{
+	struct erdma_cep *cep = sk_to_cep(sk);
+
+	if (cep) {
+		cep->sk_error = sk->sk_err;
+		cep->sk_error_report(sk);
+	}
+}
+
+static void erdma_cm_llp_state_change(struct sock *sk)
+{
+	struct erdma_cep *cep;
+	struct socket *s;
+	void (*orig_state_change)(struct sock *sk);
+
+	read_lock(&sk->sk_callback_lock);
+
+	cep = sk_to_cep(sk);
+	if (!cep) {
+		read_unlock(&sk->sk_callback_lock);
+		return;
+	}
+	orig_state_change = cep->sk_state_change;
+
+	s = sk->sk_socket;
+
+	switch (sk->sk_state) {
+	case TCP_ESTABLISHED:
+		if (cep->state == ERDMA_EPSTATE_CONNECTING)
+			erdma_cm_queue_work(cep, ERDMA_CM_WORK_CONNECTED);
+		else
+			erdma_cm_queue_work(cep, ERDMA_CM_WORK_ACCEPT);
+		break;
+	case TCP_CLOSE:
+	case TCP_CLOSE_WAIT:
+		if (cep->state != ERDMA_EPSTATE_LISTENING)
+			erdma_cm_queue_work(cep, ERDMA_CM_WORK_PEER_CLOSE);
+		break;
+	default:
+		break;
+	}
+	read_unlock(&sk->sk_callback_lock);
+	orig_state_change(sk);
+}
+
+
+static int kernel_bindconnect(struct socket *s,
+			      struct sockaddr *laddr, int laddrlen,
+			      struct sockaddr *raddr, int raddrlen, int flags)
+{
+	int err;
+	struct sock *sk = s->sk;
+
+	/*
+	 * Make address available again asap.
+	 */
+	sock_set_reuseaddr(s->sk);
+
+	err = s->ops->bind(s, laddr, laddrlen);
+	if (err < 0) {
+		pr_info("try port (%u) failed\n", ((struct sockaddr_in *)laddr)->sin_port);
+		/* Try to alloc port, not use RDMA port. */
+		((struct sockaddr_in *)laddr)->sin_port = 0;
+		err = s->ops->bind(s, laddr, laddrlen);
+		if (err < 0)
+			goto done;
+		pr_info("alloc source port %u.\n", inet_sk(sk)->inet_num);
+	}
+
+	err = s->ops->connect(s, raddr, raddrlen, flags);
+	if (err < 0)
+		goto done;
+
+	err = s->ops->getname(s, laddr, 0);
+done:
+	return err;
+}
+
+
+int erdma_connect(struct iw_cm_id *id, struct iw_cm_conn_param *params)
+{
+	struct erdma_dev *dev = to_edev(id->device);
+	struct erdma_qp *qp;
+	struct erdma_cep *cep = NULL;
+	struct socket *s = NULL;
+	struct sockaddr *laddr, *raddr;
+	u16 pd_len = params->private_data_len;
+	int ret;
+
+	if (pd_len > MPA_MAX_PRIVDATA)
+		return -EINVAL;
+
+	qp = find_qp_by_qpn(dev, params->qpn);
+	if (!qp)
+		return -ENOENT;
+
+	laddr = (struct sockaddr *)&id->m_local_addr;
+	raddr = (struct sockaddr *)&id->m_remote_addr;
+
+	qp->attrs.sip = ntohl(to_sockaddr_in(id->local_addr).sin_addr.s_addr);
+	qp->attrs.origin_sport = ntohs(to_sockaddr_in(id->local_addr).sin_port);
+	qp->attrs.dip = ntohl(to_sockaddr_in(id->remote_addr).sin_addr.s_addr);
+	qp->attrs.dport = ntohs(to_sockaddr_in(id->m_remote_addr).sin_port);
+
+	ret = sock_create(AF_INET, SOCK_STREAM, IPPROTO_TCP, &s);
+	if (ret < 0)
+		goto error_put_qp;
+
+	cep = erdma_cep_alloc(dev);
+	if (!cep) {
+		ret = -ENOMEM;
+		goto error_release_sock;
+	}
+
+	erdma_cep_set_inuse(cep);
+
+	/* Associate QP with CEP */
+	erdma_cep_get(cep);
+	qp->cep = cep;
+	erdma_qp_get(qp);
+	cep->qp = qp;
+
+	/* Associate cm_id with CEP */
+	id->add_ref(id);
+	cep->cm_id = id;
+
+	ret = erdma_cm_alloc_work(cep, 6);
+	if (ret != 0) {
+		ret = -ENOMEM;
+		goto error_release_cep;
+	}
+
+	cep->ird = params->ird;
+	cep->ord = params->ord;
+	cep->state = ERDMA_EPSTATE_CONNECTING;
+	cep->is_connecting = true;
+
+	erdma_cep_socket_assoc(cep, s);
+
+	cep->pd_len = pd_len;
+	cep->private_storage = kmalloc(pd_len, GFP_KERNEL);
+	if (!cep->private_storage) {
+		ret = -ENOMEM;
+		goto error_disasssoc;
+	}
+
+	memcpy(cep->private_storage, params->private_data, params->private_data_len);
+
+	ret = kernel_bindconnect(s, laddr, sizeof(*laddr), raddr,
+				sizeof(*raddr), O_NONBLOCK);
+	if (ret != -EINPROGRESS && ret != 0) {
+		goto error_disasssoc;
+	} else if (ret == 0) {
+		ret = erdma_cm_queue_work(cep, ERDMA_CM_WORK_CONNECTED);
+	} else {
+		ret = erdma_cm_queue_work(cep, ERDMA_CM_WORK_CONNECTTIMEOUT);
+		if (ret)
+			goto error_disasssoc;
+	}
+
+	erdma_cep_set_free(cep);
+	return 0;
+
+error_disasssoc:
+	kfree(cep->private_storage);
+	cep->private_storage = NULL;
+	cep->pd_len = 0;
+
+	erdma_socket_disassoc(s);
+
+error_release_cep:
+	/* disassoc with cm_id */
+	cep->cm_id = NULL;
+	id->rem_ref(id);
+
+	/* disassoc with qp */
+	qp->cep = NULL;
+	erdma_cep_put(cep);
+	cep->qp = NULL;
+	erdma_qp_put(qp);
+
+	cep->state = ERDMA_EPSTATE_CLOSED;
+
+	erdma_cep_set_free(cep);
+
+	/* release the cep. */
+	erdma_cep_put(cep);
+
+error_release_sock:
+	if (s)
+		sock_release(s);
+error_put_qp:
+	erdma_qp_put(qp);
+
+	return ret;
+}
+
+int erdma_accept(struct iw_cm_id *id, struct iw_cm_conn_param *params)
+{
+	struct erdma_dev *dev = to_edev(id->device);
+	struct erdma_cep *cep = (struct erdma_cep *)id->provider_data;
+	struct erdma_qp *qp;
+	struct erdma_qp_attrs qp_attrs;
+	int ret;
+
+	erdma_cep_set_inuse(cep);
+	erdma_cep_put(cep);
+
+	/* Free lingering inbound private data */
+	if (cep->mpa.hdr.params.pd_len) {
+		cep->mpa.hdr.params.pd_len = 0;
+		kfree(cep->mpa.pdata);
+		cep->mpa.pdata = NULL;
+	}
+	erdma_cancel_mpatimer(cep);
+
+	if (cep->state != ERDMA_EPSTATE_RECVD_MPAREQ) {
+		if (cep->state == ERDMA_EPSTATE_CLOSED) {
+			erdma_cep_set_free(cep);
+			erdma_cep_put(cep);
+			return -ECONNRESET;
+		}
+		return -EBADFD;
+	}
+
+	qp = find_qp_by_qpn(dev, params->qpn);
+	if (!qp)
+		return -ENOENT;
+
+	down_write(&qp->state_lock);
+	if (qp->attrs.state > ERDMA_QP_STATE_RTS) {
+		ret = -EINVAL;
+		up_write(&qp->state_lock);
+		goto error;
+	}
+
+	if (params->ord > dev->attrs.max_ord ||
+	    params->ird > dev->attrs.max_ord) {
+		ret = -EINVAL;
+		up_write(&qp->state_lock);
+		goto error;
+	}
+
+	if (params->private_data_len > MPA_MAX_PRIVDATA) {
+		ret = -EINVAL;
+		up_write(&qp->state_lock);
+		goto error;
+	}
+
+	cep->cm_id = id;
+	id->add_ref(id);
+
+	memset(&qp_attrs, 0, sizeof(qp_attrs));
+	qp_attrs.orq_size = params->ord;
+	qp_attrs.irq_size = params->ird;
+	qp_attrs.llp_stream_handle = cep->llp.sock;
+
+	qp_attrs.state = ERDMA_QP_STATE_RTS;
+
+	qp->attrs.sip = ntohl(cep->llp.laddr.sin_addr.s_addr);
+	qp->attrs.origin_sport = ntohs(cep->llp.laddr.sin_port);
+	qp->attrs.dip = ntohl(cep->llp.raddr.sin_addr.s_addr);
+	qp->attrs.dport = ntohs(cep->llp.raddr.sin_port);
+	qp->attrs.sport = ntohs(cep->llp.laddr.sin_port);
+
+	/* Associate QP with CEP */
+	erdma_cep_get(cep);
+	qp->cep = cep;
+
+	erdma_qp_get(qp);
+	cep->qp = qp;
+
+	cep->state = ERDMA_EPSTATE_RDMA_MODE;
+
+	qp->qp_type = ERDMA_QP_TYPE_SERVER;
+	qp->private_data_len = params->private_data_len;
+	qp->cc_method = __mpa_rr_cc(cep->mpa.hdr.params.bits) == qp->dev->cc_method ?
+		qp->dev->cc_method : COMPROMISE_CC;
+
+	/* move to rts */
+	ret = erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE |
+				       ERDMA_QP_ATTR_LLP_HANDLE |
+				       ERDMA_QP_ATTR_MPA);
+	up_write(&qp->state_lock);
+
+	if (ret)
+		goto error;
+
+	__mpa_rr_set_cc(&cep->mpa.hdr.params.bits, qp->dev->cc_method);
+	memcpy(&cep->mpa.hdr.key[12], (u32 *)&QP_ID(qp), 4);
+	ret = erdma_send_mpareqrep(cep, params->private_data,
+				params->private_data_len);
+
+	if (!ret) {
+		ret = erdma_cm_upcall(cep, IW_CM_EVENT_ESTABLISHED, 0);
+		if (ret)
+			goto error;
+
+		erdma_cep_set_free(cep);
+
+		return 0;
+	}
+
+error:
+	erdma_socket_disassoc(cep->llp.sock);
+	sock_release(cep->llp.sock);
+	cep->llp.sock = NULL;
+
+	cep->state = ERDMA_EPSTATE_CLOSED;
+
+	if (cep->cm_id) {
+		cep->cm_id->rem_ref(id);
+		cep->cm_id = NULL;
+	}
+	if (qp->cep) {
+		erdma_cep_put(cep);
+		qp->cep = NULL;
+	}
+
+	cep->qp = NULL;
+	erdma_qp_put(qp);
+
+	erdma_cep_set_free(cep);
+	erdma_cep_put(cep);
+
+	return ret;
+}
+
+/*
+ * erdma_reject()
+ *
+ * Local connection reject case. Send private data back to peer,
+ * close connection and dereference connection id.
+ */
+int erdma_reject(struct iw_cm_id *id, const void *pdata, u8 plen)
+{
+	struct erdma_cep	*cep = (struct erdma_cep *)id->provider_data;
+
+	erdma_cep_set_inuse(cep);
+	erdma_cep_put(cep);
+
+	erdma_cancel_mpatimer(cep);
+
+	if (cep->state != ERDMA_EPSTATE_RECVD_MPAREQ) {
+		if (cep->state == ERDMA_EPSTATE_CLOSED) {
+			erdma_cep_set_free(cep);
+			erdma_cep_put(cep); /* should be last reference */
+
+			return -ECONNRESET;
+		}
+		return -EBADFD;
+	}
+
+	if (__mpa_rr_revision(cep->mpa.hdr.params.bits) == MPA_REVISION_1) {
+		cep->mpa.hdr.params.bits |= MPA_RR_FLAG_REJECT; /* reject */
+		(void)erdma_send_mpareqrep(cep, pdata, plen);
+	}
+	erdma_socket_disassoc(cep->llp.sock);
+	sock_release(cep->llp.sock);
+	cep->llp.sock = NULL;
+
+	cep->state = ERDMA_EPSTATE_CLOSED;
+
+	erdma_cep_set_free(cep);
+	erdma_cep_put(cep);
+
+	return 0;
+}
+
+int erdma_create_listen(struct iw_cm_id *id, int backlog)
+{
+	struct socket *s;
+	struct erdma_cep *cep = NULL;
+	int ret = 0;
+	struct erdma_dev *dev = to_edev(id->device);
+	int addr_family = id->local_addr.ss_family;
+
+	if (addr_family != AF_INET)
+		return -EAFNOSUPPORT;
+
+	ret = sock_create(addr_family, SOCK_STREAM, IPPROTO_TCP, &s);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Allow binding local port when still in TIME_WAIT from last close.
+	 */
+	sock_set_reuseaddr(s->sk);
+
+	if (addr_family == AF_INET) {
+		struct sockaddr_in *laddr = &to_sockaddr_in(id->local_addr);
+		u8 *l_ip, *r_ip;
+
+		l_ip = (u8 *) &to_sockaddr_in(id->local_addr).sin_addr.s_addr;
+		r_ip = (u8 *) &to_sockaddr_in(id->remote_addr).sin_addr.s_addr;
+
+		/* For wildcard addr, limit binding to current device only */
+		if (ipv4_is_zeronet(laddr->sin_addr.s_addr))
+			s->sk->sk_bound_dev_if = dev->netdev->ifindex;
+
+		ret = s->ops->bind(s, (struct sockaddr *)laddr, sizeof(struct sockaddr_in));
+	} else {
+		ret = -EAFNOSUPPORT;
+		goto error;
+	}
+
+	if (ret != 0)
+		goto error;
+
+	cep = erdma_cep_alloc(dev);
+	if (!cep) {
+		ret = -ENOMEM;
+		goto error;
+	}
+	erdma_cep_socket_assoc(cep, s);
+
+	ret = erdma_cm_alloc_work(cep, backlog);
+	if (ret != 0)
+		goto error;
+
+	ret = s->ops->listen(s, backlog);
+	if (ret != 0)
+		goto error;
+
+	memcpy(&cep->llp.laddr, &id->local_addr, sizeof(cep->llp.laddr));
+	memcpy(&cep->llp.raddr, &id->remote_addr, sizeof(cep->llp.raddr));
+
+	cep->cm_id = id;
+	id->add_ref(id);
+
+	if (!id->provider_data) {
+		id->provider_data = kmalloc(sizeof(struct list_head), GFP_KERNEL);
+		if (!id->provider_data) {
+			ret = -ENOMEM;
+			goto error;
+		}
+		INIT_LIST_HEAD((struct list_head *)id->provider_data);
+	}
+
+	list_add_tail(&cep->listenq, (struct list_head *)id->provider_data);
+	cep->state = ERDMA_EPSTATE_LISTENING;
+
+	return 0;
+
+error:
+	if (cep) {
+		erdma_cep_set_inuse(cep);
+
+		if (cep->cm_id) {
+			cep->cm_id->rem_ref(cep->cm_id);
+			cep->cm_id = NULL;
+		}
+		cep->llp.sock = NULL;
+		erdma_socket_disassoc(s);
+		cep->state = ERDMA_EPSTATE_CLOSED;
+
+		erdma_cep_set_free(cep);
+		erdma_cep_put(cep);
+	}
+	sock_release(s);
+
+	return ret;
+}
+
+static void erdma_drop_listeners(struct iw_cm_id *id)
+{
+	struct list_head	*p, *tmp;
+	/*
+	 * In case of a wildcard rdma_listen on a multi-homed device,
+	 * a listener's IWCM id is associated with more than one listening CEP.
+	 */
+	list_for_each_safe(p, tmp, (struct list_head *)id->provider_data) {
+		struct erdma_cep *cep = list_entry(p, struct erdma_cep, listenq);
+
+		list_del(p);
+
+		erdma_cep_set_inuse(cep);
+
+		if (cep->cm_id) {
+			cep->cm_id->rem_ref(cep->cm_id);
+			cep->cm_id = NULL;
+		}
+		if (cep->llp.sock) {
+			erdma_socket_disassoc(cep->llp.sock);
+			sock_release(cep->llp.sock);
+			cep->llp.sock = NULL;
+		}
+		cep->state = ERDMA_EPSTATE_CLOSED;
+		erdma_cep_set_free(cep);
+		erdma_cep_put(cep);
+	}
+}
+
+int erdma_destroy_listen(struct iw_cm_id *id)
+{
+	if (!id->provider_data)
+		return 0;
+
+	erdma_drop_listeners(id);
+	kfree(id->provider_data);
+	id->provider_data = NULL;
+
+	return 0;
+}
+
+int erdma_cm_init(void)
+{
+	erdma_cm_wq = create_singlethread_workqueue("erdma_cm_wq");
+	if (!erdma_cm_wq)
+		return -ENOMEM;
+
+	return 0;
+}
+
+void erdma_cm_exit(void)
+{
+	if (erdma_cm_wq) {
+		flush_workqueue(erdma_cm_wq);
+		destroy_workqueue(erdma_cm_wq);
+	}
+}