Skip to contentSkip to conversation

rsockets: acknowledge completion queue events in batch

PATCH Superseded Sreedhar Kodali 2 participants 2 messages Sep 18, 2014 – Sep 25, 2014 1 message not in the archive InfiniBand

v5 of 6 Superseded Latest is v6 →

lore
Share & tools
lore

Sreedhar Kodali

author
#0 lore raw

PATCH v5 2/4 rsockets: acknowledge completion queue events in batch

Note: This patch is being resent as suggested by Sean
Also attached the patch file for convenience

From: Sreedhar Kodali <[email protected]>
Date: Thu Sep 18 14:29:48 2014 +0530

Perform completion event acknowledgments in batches instead of individually to minimze locking overheads. Size of the completion queue decides the size of a batch.

Signed-off-by Sreedhar Kodali <[email protected]>

---

src/rsocket.c +6 -1
index 00d9a18..39c35cf 100644
@@ -381,6 +381,7 @@ struct rsocket {
        dlist_entry       iomap_list;
        dlist_entry       iomap_queue;
        int               iomap_pending;
+       int           unack_cqe;
 };
 
 #define DS_UDP_TAG 0x55555555

@@ -599,6 +600,7 @@ static struct rsocket *rs_alloc(struct rsocket *inherited_rs, int type)
                        rs->target_iomap_size = def_iomap_size;
                }
        }
+       rs->unack_cqe = 0;
        fastlock_init(&rs->slock);
        fastlock_init(&rs->rlock);
        fastlock_init(&rs->cq_lock);

@@ -990,8 +992,11 @@ static void rs_free(struct rsocket *rs)
 
        if (rs->cm_id) {
                rs_free_iomappings(rs);
-               if (rs->cm_id->qp)
+               if (rs->cm_id->qp) {
+                       if (rs->unack_cqe > 0)
+                               ibv_ack_cq_events(rs->cm_id->recv_cq, rs->unack_cqe);
                        rdma_destroy_qp(rs->cm_id);
+               }
                rdma_destroy_id(rs->cm_id);
        }

@@ -1965,7 +1970,11 @@ static int rs_get_cq_event(struct rsocket *rs)
 
        ret = ibv_get_cq_event(rs->cm_id->recv_cq_channel, &cq, 

&context);
        if (!ret) {
-               ibv_ack_cq_events(rs->cm_id->recv_cq, 1);
+               rs->unack_cqe += 1;
+               if (rs->unack_cqe == rs->sq_size + rs->rq_size) {
+                       ibv_ack_cq_events(rs->cm_id->recv_cq, rs->unack_cqe);
+                       rs->unack_cqe = 0;
+               }
                rs->cq_armed = 0;
        } else if (errno != EAGAIN) {
                rs->state = rs_error;

commit 490a2aa74e7dcdefe684e850d65b68bde3516ec4

Author: Sreedhar Kodali <[email protected]>
Date:   Thu Sep 18 14:29:48 2014 +0530

Perform completion event acknowledgments in batches instead of individually to minimze locking overheads. Size of the completion queue decides the size of a batch.

Signed-off-by Sreedhar Kodali <[email protected]>

---

src/rsocket.c +11 -2
index 00d9a18..39c35cf 100644
@@ -381,6 +381,7 @@ struct rsocket {
 	dlist_entry	  iomap_list;
 	dlist_entry	  iomap_queue;
 	int		  iomap_pending;
+	int	      unack_cqe;
 };
 
 #define DS_UDP_TAG 0x55555555

@@ -599,6 +600,7 @@ static struct rsocket *rs_alloc(struct rsocket *inherited_rs, int type)
 			rs->target_iomap_size = def_iomap_size;
 		}
 	}
+	rs->unack_cqe = 0;
 	fastlock_init(&rs->slock);
 	fastlock_init(&rs->rlock);
 	fastlock_init(&rs->cq_lock);

@@ -990,8 +992,11 @@ static void rs_free(struct rsocket *rs)
 
 	if (rs->cm_id) {
 		rs_free_iomappings(rs);
-		if (rs->cm_id->qp)
+		if (rs->cm_id->qp) {
+			if (rs->unack_cqe > 0)
+				ibv_ack_cq_events(rs->cm_id->recv_cq, rs->unack_cqe);
 			rdma_destroy_qp(rs->cm_id);
+		}
 		rdma_destroy_id(rs->cm_id);
 	}
 

@@ -1965,7 +1970,11 @@ static int rs_get_cq_event(struct rsocket *rs)
 
 	ret = ibv_get_cq_event(rs->cm_id->recv_cq_channel, &cq, &context);
 	if (!ret) {
-		ibv_ack_cq_events(rs->cm_id->recv_cq, 1);
+		rs->unack_cqe += 1;
+		if (rs->unack_cqe == rs->sq_size + rs->rq_size) {
+			ibv_ack_cq_events(rs->cm_id->recv_cq, rs->unack_cqe);
+			rs->unack_cqe = 0;
+		}
 		rs->cq_armed = 0;
 	} else if (errno != EAGAIN) {
 		rs->state = rs_error;

A message that is not in the archive (sent off-list or not yet received): 18bdfd1d731439cbe604263b37379cb6-FJGp5E75HVmZamtmwQBW5tBPR1lH4CV8@public.gmane.org