no IBV_SEND_INLINE thus memory registration required on QLogic/Intel HCA

index 4c4a057..5b433ab 100644
@@ -386,6 +386,22 @@ int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,
 		   struct ibv_qp_init_attr *qp_init_attr);
 
 /**
+ * rdma_query_qp - Query a QP.
+ * @id: RDMA identifier.
+ * @qp_init_attr: initial QP attributes.
+ * Description:
+ *  Query QP init attributes.
+ * Notes:
+ *   The rdma_cm_id must be bound to QP before calling this function.
+ *   This function should be used to check QP capabilities.
+ * See also:
+ *   rdma_create_ep, rdma_get_request, rdma_create_qp, ibv_create_qp,
+ *   ibv_query_qp
+ */
+int rdma_query_qp(struct rdma_cm_id *id,
+		  struct ibv_qp_init_attr *qp_init_attr);
+
+/**
  * rdma_destroy_qp - Deallocate a QP.
  * @id: RDMA identifier.
  * Description:

index 374844c..24fd86b 100644
@@ -1265,6 +1265,30 @@ err1:
 	return ret;
 }
 
+int rdma_query_qp(struct rdma_cm_id *id,
+		  struct ibv_qp_init_attr *qp_init_attr)
+{
+	struct ibv_qp_attr attr;
+	int ret;
+
+	memset(&attr, 0, sizeof attr);
+	memset(qp_init_attr, 0, sizeof *qp_init_attr);
+
+	ret = ibv_query_qp(id->qp, &attr, IBV_QP_CAP, qp_init_attr);
+	if (ret)
+		return ERR(ret);
+
+	assert(attr.cap.max_send_wr == qp_init_attr->cap.max_send_wr);
+	assert(attr.cap.max_recv_wr == qp_init_attr->cap.max_recv_wr);
+
+	assert(attr.cap.max_send_sge == qp_init_attr->cap.max_send_sge);
+	assert(attr.cap.max_recv_sge == qp_init_attr->cap.max_recv_sge);
+
+	assert(attr.cap.max_inline_data == qp_init_attr->cap.max_inline_data);
+
+	return 0;
+}
+
 void rdma_destroy_qp(struct rdma_cm_id *id)
 {
 	ibv_destroy_qp(id->qp);

index 7a59d97..830bdc2 100644
@@ -39,7 +39,8 @@ static char *server = "127.0.0.1";
 static char *port = "7471";
 
 struct rdma_cm_id *id;
-struct ibv_mr *mr;
+struct ibv_mr *send_mr;
+struct ibv_mr *recv_mr;
 uint8_t send_msg[16];
 uint8_t recv_msg[16];
 

@@ -71,13 +72,13 @@ static int run(void)
 		return ret;
 	}
 
-	mr = rdma_reg_msgs(id, recv_msg, 16);
-	if (!mr) {
+	recv_mr = rdma_reg_msgs(id, recv_msg, 16);
+	if (!recv_mr) {
 		printf("rdma_reg_msgs %d\n", errno);
 		return ret;
 	}
 
-	ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);
+	ret = rdma_post_recv(id, NULL, recv_msg, 16, recv_mr);
 	if (ret) {
 		printf("rdma_post_recv %d\n", errno);
 		return ret;

@@ -89,7 +90,17 @@ static int run(void)
 		return ret;
 	}
 
-	ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE);
+	if (attr.cap.max_inline_data < 16) {
+		send_mr = rdma_reg_msgs(id, send_msg, 16);
+		if (!send_mr) {
+			printf("rdma_reg_msgs %d\n", errno);
+			return ret;
+		}
+	}
+
+	ret = rdma_post_send(id, NULL, send_msg, 16,
+			     attr.cap.max_inline_data < 16 ? send_mr : NULL,
+			     attr.cap.max_inline_data < 16 ? 0 : IBV_SEND_INLINE);
 	if (ret) {
 		printf("rdma_post_send %d\n", errno);
 		return ret;

@@ -102,7 +113,9 @@ static int run(void)
 	}
 
 	rdma_disconnect(id);
-	rdma_dereg_mr(mr);
+	rdma_dereg_mr(recv_mr);
+	if (attr.cap.max_inline_data < 16)
+		rdma_dereg_mr(send_mr);
 	rdma_destroy_ep(id);
 	return 0;
 }

index 5b9e16d..459cb38 100644
@@ -39,7 +39,8 @@
 static char *port = "7471";
 
 struct rdma_cm_id *listen_id, *id;
-struct ibv_mr *mr;
+struct ibv_mr *send_mr;
+struct ibv_mr *recv_mr;
 uint8_t send_msg[16];
 uint8_t recv_msg[16];
 

@@ -83,13 +84,13 @@ static int run(void)
 		return ret;
 	}
 
-	mr = rdma_reg_msgs(id, recv_msg, 16);
-	if (!mr) {
+	recv_mr = rdma_reg_msgs(id, recv_msg, 16);
+	if (!recv_mr) {
 		printf("rdma_reg_msgs %d\n", errno);
 		return ret;
 	}
 
-	ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);
+	ret = rdma_post_recv(id, NULL, recv_msg, 16, recv_mr);
 	if (ret) {
 		printf("rdma_post_recv %d\n", errno);
 		return ret;

@@ -107,7 +108,23 @@ static int run(void)
 		return ret;
 	}
 
-	ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE);
+	ret = rdma_query_qp(id, &attr);
+	if (ret) {
+		printf("rdma_query_qp %d\n", errno);
+		return ret;
+	}
+
+	if (attr.cap.max_inline_data < 16) {
+		send_mr = rdma_reg_msgs(id, send_msg, 16);
+		if (!send_mr) {
+			printf("rdma_reg_msgs %d\n", errno);
+			return ret;
+		}
+	}
+
+	ret = rdma_post_send(id, NULL, send_msg, 16,
+			     attr.cap.max_inline_data < 16 ? send_mr : NULL,
+			     attr.cap.max_inline_data < 16 ? 0 : IBV_SEND_INLINE);
 	if (ret) {
 		printf("rdma_post_send %d\n", errno);
 		return ret;

@@ -120,7 +137,9 @@ static int run(void)
 	}
 
 	rdma_disconnect(id);
-	rdma_dereg_mr(mr);
+	rdma_dereg_mr(recv_mr);
+	if (attr.cap.max_inline_data < 16)
+		rdma_dereg_mr(send_mr);
 	rdma_destroy_ep(id);
 	rdma_destroy_ep(listen_id);
 	return 0;

95 lines not on this page · raw message

index df3e665..32a119a 100644
@@ -42,7 +42,8 @@ static char *port = "7471";
 
 static int (*run_func)();
 struct rdma_cm_id *listen_id, *id;
-struct ibv_mr *mr;
+struct ibv_mr *send_mr;
+struct ibv_mr *recv_mr;
 enum ibv_qp_type qpt = IBV_QPT_RC;
 
 #define MSG_SIZE 16

@@ -192,13 +193,13 @@ static int xrc_test(void)
 		return ret;
 	}
 
-	mr = rdma_reg_msgs(srq_id, recv_msg, sizeof recv_msg);
-	if (!mr) {
+	recv_mr = rdma_reg_msgs(srq_id, recv_msg, sizeof recv_msg);
+	if (!recv_mr) {
 		printf("ibv_reg_msgs %d\n", errno);
 		return ret;
 	}
 
-	ret = rdma_post_recv(srq_id, NULL, recv_msg, sizeof recv_msg, mr);
+	ret = rdma_post_recv(srq_id, NULL, recv_msg, sizeof recv_msg, recv_mr);
 	if (ret) {
 		printf("rdma_post_recv %d\n", errno);
 		return ret;

@@ -229,7 +230,7 @@ static int xrc_test(void)
 	rdma_ack_cm_event(event);
 	rdma_disconnect(conn_id);
 	rdma_destroy_ep(conn_id);
-	rdma_dereg_mr(mr);
+	rdma_dereg_mr(recv_mr);
 	rdma_destroy_ep(srq_id);
 	rdma_destroy_ep(listen_id);
 	return 0;

@@ -288,13 +289,13 @@ static int rc_test(void)
 		return ret;
 	}
 
-	mr = rdma_reg_msgs(id, recv_msg, sizeof recv_msg);
-	if (!mr) {
+	recv_mr = rdma_reg_msgs(id, recv_msg, sizeof recv_msg);
+	if (!recv_mr) {
 		printf("rdma_reg_msgs %d\n", errno);
 		return ret;
 	}
 
-	ret = rdma_post_recv(id, NULL, recv_msg, sizeof recv_msg, mr);
+	ret = rdma_post_recv(id, NULL, recv_msg, sizeof recv_msg, recv_mr);
 	if (ret) {
 		printf("rdma_post_recv %d\n", errno);
 		return ret;

@@ -312,7 +313,23 @@ static int rc_test(void)
 		return ret;
 	}
 
-	ret = rdma_post_send(id, NULL, send_msg, sizeof send_msg, NULL, IBV_SEND_INLINE);
+	ret = rdma_query_qp(id, &attr);
+	if (ret) {
+		printf("rdma_query_qp %d\n", errno);
+		return ret;
+	}
+
+	if (attr.cap.max_inline_data < sizeof send_msg) {
+		send_mr = rdma_reg_msgs(id, send_msg, sizeof send_msg);
+		if (!send_mr) {
+			printf("rdma_reg_msgs %d\n", errno);
+			return ret;
+		}
+	}
+
+	ret = rdma_post_send(id, NULL, send_msg, sizeof send_msg,
+			     attr.cap.max_inline_data < sizeof send_msg ? send_mr : NULL,
+			     attr.cap.max_inline_data < sizeof send_msg ? 0 : IBV_SEND_INLINE);
 	if (ret) {
 		printf("rdma_post_send %d\n", errno);
 		return ret;

@@ -325,7 +342,9 @@ static int rc_test(void)
 	}
 
 	rdma_disconnect(id);
-	rdma_dereg_mr(mr);
+	rdma_dereg_mr(recv_mr);
+	if (attr.cap.max_inline_data < sizeof send_msg)
+		rdma_dereg_mr(send_mr);
 	rdma_destroy_ep(id);
 	rdma_destroy_ep(listen_id);
 	return 0;