Skip to contentSkip to conversation

staging/rdma/hfi1: add common routine for queuing acks

PATCH New Mike Marciniszyn 2 participants 2 messages Nov 10, 2015

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/2 staging/rdma/hfi1: add common routine for queuing acks

This patch is a prelimary patch required to
coalesce acks.

The routine to "schedule" a QP for sending a NAK is now centralized in rc_defer_ack(). The flag is changed for clarity since the all acks will potentially use the deferral mechanism.

---
 drivers/staging/rdma/hfi1/driver.c |    4 ++-
 drivers/staging/rdma/hfi1/rc.c     |   42 +++++++++++++-----------------------
 drivers/staging/rdma/hfi1/verbs.h  |   12 ++++++----
 3 files changed, 24 insertions(+), 34 deletions(-)
drivers/staging/rdma/hfi1/driver.c +2 -2
index ce69141..e8e0f55 100644
@@ -721,8 +721,8 @@ static inline void process_rcv_qp_work(struct hfi1_packet *packet)
 	 */
 	list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) {
 		list_del_init(&qp->rspwait);
-		if (qp->r_flags & HFI1_R_RSP_NAK) {
-			qp->r_flags &= ~HFI1_R_RSP_NAK;
+		if (qp->r_flags & HFI1_R_RSP_DEFERED_ACK) {
+			qp->r_flags &= ~HFI1_R_RSP_DEFERED_ACK;
 			hfi1_send_rc_ack(rcd, qp, 0);
 		}
 		if (qp->r_flags & HFI1_R_RSP_SEND) {
drivers/staging/rdma/hfi1/rc.c +15 -27

78 lines not on this page · raw message

drivers/staging/rdma/hfi1/verbs.h +7 -5
index e4a8a0d..c5e6f47 100644
@@ -547,11 +547,13 @@ struct hfi1_qp {
 /*
  * Bit definitions for r_flags.
  */
-#define HFI1_R_REUSE_SGE 0x01
-#define HFI1_R_RDMAR_SEQ 0x02
-#define HFI1_R_RSP_NAK   0x04
-#define HFI1_R_RSP_SEND  0x08
-#define HFI1_R_COMM_EST  0x10
+#define HFI1_R_REUSE_SGE       0x01
+#define HFI1_R_RDMAR_SEQ       0x02
+/* defer ack until end of interrupt session */
+#define HFI1_R_RSP_DEFERED_ACK 0x04
+/* relay ack to send engine */
+#define HFI1_R_RSP_SEND        0x08
+#define HFI1_R_COMM_EST        0x10
 
 /*
  * Bit definitions for s_flags.

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

#1 lore raw

PATCH 2/2 staging/rdma/hfi1: add ACK coalescing logic

Implement ACK coalesing logic using a 8 bit counter.

The algorithm is send pio ack when:
- fecn present
- this is the first packet in an interrupt session
- counter is >= HFI1_PSN_CREDIT

Otherwise the ack is defered.

---
 drivers/staging/rdma/hfi1/qp.c    |    1 +
 drivers/staging/rdma/hfi1/rc.c    |   29 +++++++++++++++++++++++++++--
 drivers/staging/rdma/hfi1/verbs.h |    7 ++++---
 3 files changed, 32 insertions(+), 5 deletions(-)
drivers/staging/rdma/hfi1/qp.c +1 -0
index df1fa56..8f867ba 100644
@@ -378,6 +378,7 @@ static void reset_qp(struct hfi1_qp *qp, enum ib_qp_type type)
 	}
 	qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
 	qp->r_nak_state = 0;
+	qp->r_adefered = 0;
 	qp->r_aflags = 0;
 	qp->r_flags = 0;
 	qp->s_head = 0;
drivers/staging/rdma/hfi1/rc.c +27 -2
index 6fe0104..0193497 100644
@@ -1618,6 +1618,17 @@ static inline void rc_defered_ack(struct hfi1_ctxtdata *rcd,
 	}
 }
 
+static inline void rc_cancel_ack(struct hfi1_qp *qp)
+{
+	qp->r_adefered = 0;
+	if (list_empty(&qp->rspwait))
+		return;
+	list_del_init(&qp->rspwait);
+	qp->r_flags &= ~HFI1_R_RSP_DEFERED_ACK;
+	if (atomic_dec_and_test(&qp->refcount))
+		wake_up(&qp->wait);
+}
+
 /**
  * rc_rcv_error - process an incoming duplicate or error RC packet
  * @ohdr: the other headers for this packet
@@ -2335,8 +2346,22 @@ send_last:
 	qp->r_ack_psn = psn;
 	qp->r_nak_state = 0;
 	/* Send an ACK if requested or required. */
-	if (psn & (1 << 31))
-		goto send_ack;
+	if (psn & IB_BTH_REQ_ACK) {
+		if (packet->numpkt == 0) {
+			rc_cancel_ack(qp);
+			goto send_ack;
+		}
+		if (qp->r_adefered >= HFI1_PSN_CREDIT) {
+			rc_cancel_ack(qp);
+			goto send_ack;
+		}
+		if (unlikely(is_fecn)) {
+			rc_cancel_ack(qp);
+			goto send_ack;
+		}
+		qp->r_adefered++;
+		rc_defered_ack(rcd, qp);
+	}
 	return;
 
 rnr_nak:
drivers/staging/rdma/hfi1/verbs.h +4 -3
index c5e6f47..6d2012b 100644
@@ -120,9 +120,9 @@ struct hfi1_packet;
 
 #define HFI1_VENDOR_IPG		cpu_to_be16(0xFFA0)
 
-#define IB_BTH_REQ_ACK		(1 << 31)
-#define IB_BTH_SOLICITED	(1 << 23)
-#define IB_BTH_MIG_REQ		(1 << 22)
+#define IB_BTH_REQ_ACK		BIT(31)
+#define IB_BTH_SOLICITED	BIT(23)
+#define IB_BTH_MIG_REQ		BIT(22)
 
 #define IB_GRH_VERSION		6
 #define IB_GRH_VERSION_MASK	0xF
@@ -484,6 +484,7 @@ struct hfi1_qp {
 	u32 r_psn;              /* expected rcv packet sequence number */
 	u32 r_msn;              /* message sequence number */
 
+	u8 r_adefered;         /* number of acks defered */
 	u8 r_state;             /* opcode of last packet received */
 	u8 r_flags;
 	u8 r_head_ack_queue;    /* index into s_ack_queue[] */