asynchronous operation with poll()

  struct ibv_mr *mr;
  int ret;
  int send_buf_num = 0;
  int recv_buf_num = 0;

  #define NUM_BUFFERS 20
  #define SIZE 1024*1024
  uint8_t *buffer = (uint8_t*)malloc(SIZE * NUM_BUFFERS * 2);
  uint8_t *send_msg[NUM_BUFFERS];
  uint8_t *recv_msg[NUM_BUFFERS];

  for(int i=0; i<NUM_BUFFERS; i++) {
    send_msg[i] = buffer + (i*SIZE);
    recv_msg[i] = buffer + ((i+NUM_BUFFERS) * SIZE);
  }

  //--------------------------------------------------------------------
  // setup
  fprintf(stderr, "rdma_reg_msgs\n");
  mr = rdma_reg_msgs(cm_id, buffer, SIZE*NUM_BUFFERS*2);
  if (!mr) {
    perror("rdma_reg_msgs");
  }

  //prepare to for the first receive before connecting
  for(int i=0; i<10; i++) {
    fprintf(stderr, "rdma_post_recv\n");
    ret = rdma_post_recv(cm_id, NULL, recv_msg[recv_buf_num++], SIZE, mr);
    recv_buf_num %= NUM_BUFFERS;
    if (ret) {
      perror("rdma_post_recv");
    }
  }

  //connect
  fprintf(stderr, "rdma_connect\n");
  ret = rdma_connect(cm_id, NULL);
  if (ret) {
    perror("rdma_connect");
  }

  const int NUM_FDS = 4;

  const int POLL_CM = 0;
  const int POLL_RECV_CQ = 1;
  const int POLL_SEND_CQ = 2;
  const int POLL_WAKE = 3;
  struct pollfd fds[NUM_FDS];

//prime notification of events on the recv completion queue ibv_req_notify_cq(cm_id->recv_cq, 0);
//

  //--------------------------------------------------------------------
  // main loop
  while(ret == 0)
  {
    memset(fds, 0, sizeof(pollfd) * NUM_FDS);
    fds[POLL_CM].fd = cm_channel->fd;
    fds[POLL_CM].events = POLLIN;

    fds[POLL_RECV_CQ].fd = cm_id->recv_cq_channel->fd;
    fds[POLL_RECV_CQ].events = POLLIN;

    fds[POLL_SEND_CQ].fd = cm_id->send_cq_channel->fd;
    fds[POLL_SEND_CQ].events = POLLIN;

    fds[POLL_WAKE].fd = wake_fds[0];
    fds[POLL_WAKE].events = POLLIN;

    int nready = poll(fds, NUM_FDS, -1);
    if(nready < 0) {
      perror("poll");
    }

    if(fds[POLL_CM].revents & POLLIN) {
      struct rdma_cm_event *cm_event;
      ret = rdma_get_cm_event(cm_channel, &cm_event);
      if(ret) {
        perror("client connection rdma_get_cm_event");
      }
      fprintf(stderr, "Got cm event %s\n", rdma_event_str(cm_event->event));

      if(cm_event->event == RDMA_CM_EVENT_ESTABLISHED) {
        //send as soon as we are connected
        ibv_req_notify_cq(cm_id->send_cq, 0);
        ret = rdma_post_send(cm_id, NULL, send_msg[send_buf_num], SIZE, mr, 0);
        send_buf_num++;
        send_buf_num %= NUM_BUFFERS;
        if (ret) {
          perror("rdma_post_send");
        }
      }

      int finish=0;
      if(cm_event->event == RDMA_CM_EVENT_DISCONNECTED ||
         cm_event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
          finish = 1;

      rdma_ack_cm_event(cm_event);
      if(finish) {
        goto out;
      }
    }

    //if the send completed
    if(fds[POLL_SEND_CQ].revents & POLLIN) {
      struct ibv_cq *cq;
      struct ibv_wc wc[10];
      void *context;
      int num_send = ibv_poll_cq(cm_id->send_cq, 10, &wc[0]);

      if(num_send == 0) fprintf(stderr, ".");

      for(int i=0; i<num_send; i++) {
        fprintf(stderr,"Got SEND CQ event : %d of %d %s\n", i, num_send, ibv_wc_status_str(wc[i].status));
        ibv_get_cq_event(cm_id->send_cq_channel, &cq, &context);
        assert(cq == cm_id->send_cq);

        //our send completed, send some more right away
        fprintf(stderr, "rdma_post_send\n");
        ret = rdma_post_send(cm_id, NULL, send_msg[send_buf_num++], SIZE, mr, 0);
        send_buf_num %= NUM_BUFFERS;
        if (ret) {
          perror("rdma_post_send");
        }
      }

      //expensive call, ack all received events together
      ibv_ack_cq_events(cm_id->send_cq, num_send);
      ibv_req_notify_cq(cm_id->send_cq, 0);
    }

    //if the receive completed, prepare to receive more
    if(fds[POLL_RECV_CQ].revents & POLLIN) {
      struct ibv_cq *cq;
      struct ibv_wc wc[10];
      void *context;
      int num_recv=ibv_poll_cq(cm_id->recv_cq, 10, &wc[0]);

      for(int i=0; i<num_recv; i++) {
        fprintf(stderr,"Got RECV CQ event : %d of %d %s\n", i, num_recv, ibv_wc_status_str(wc[i].status));
        ibv_get_cq_event(cm_id->recv_cq_channel, &cq, &context);
        assert(cq == cm_id->recv_cq);

        //we received some payload, prepare to receive more
        fprintf(stderr, "rdma_post_recv\n");
        ret = rdma_post_recv(cm_id, NULL, recv_msg[recv_buf_num++], SIZE, mr);
        recv_buf_num %= NUM_BUFFERS;
        if (ret) {
           perror("rdma_post_recv");
        }
      }

      //expensive call, ack all received events together
      ibv_ack_cq_events(cm_id->recv_cq, num_recv);
      ibv_req_notify_cq(cm_id->recv_cq, 0);
    }

    if(fds[POLL_WAKE].revents & POLLIN) {
      fprintf(stderr, "poll WAKE\n");
      char buffer[1];
      int nread = read(wake_fds[0], buffer, 1);
      fprintf(stderr, "Got Wake event %d\n", nread);
      goto out;
    }

  }

out:
rdma_disconnect(cm_id);
rdma_dereg_mr(mr);
rdma_destroy_ep(cm_id);

  free(buffer);
  fprintf(stderr, "poll: client completed\n");

I have a client and server test program to explore fully asynchronous communication written as close to a conventional sockets application as possible and am encountering difficulty.

Quoted 6 lines

const int NUM_FDS = 4;

const int POLL_CM = 0;
const int POLL_RECV_CQ = 1;
const int POLL_SEND_CQ = 2;
const int POLL_WAKE = 3;

Quoted 2 lines

//prime notification of events on the recv completion queue ibv_req_notify_cq(cm_id->recv_cq, 0);

Quoted 6 lines

while(ret == 0)

 {
   memset(fds, 0, sizeof(pollfd) * NUM_FDS);
   fds[POLL_CM].fd = cm_channel->fd;
   fds[POLL_CM].events = POLLIN;

Quoted 8 lines

   fds[POLL_RECV_CQ].fd = cm_id->recv_cq_channel->fd;
   fds[POLL_RECV_CQ].events = POLLIN;

   fds[POLL_SEND_CQ].fd = cm_id->send_cq_channel->fd;
   fds[POLL_SEND_CQ].events = POLLIN;

   fds[POLL_WAKE].fd = wake_fds[0];
   fds[POLL_WAKE].events = POLLIN;

Quoted 11 lines

   if(fds[POLL_CM].revents & POLLIN) {
     struct rdma_cm_event *cm_event;
     ret = rdma_get_cm_event(cm_channel, &cm_event);
     if(ret) {
       perror("client connection rdma_get_cm_event");
     }
     fprintf(stderr, "Got cm event %s\n", rdma_event_str(cm_event->event));

     if(cm_event->event == RDMA_CM_EVENT_ESTABLISHED) {
       //send as soon as we are connected
       ibv_req_notify_cq(cm_id->send_cq, 0);

Quoted 7 lines

   //if the send completed
   if(fds[POLL_SEND_CQ].revents & POLLIN) {
     struct ibv_cq *cq;
     struct ibv_wc wc[10];
     void *context;
     int num_send = ibv_poll_cq(cm_id->send_cq, 10, &wc[0]);
     if(num_send == 0) fprintf(stderr, ".");

Quoted 3 lines

     for(int i=0; i<num_send; i++) {
       fprintf(stderr,"Got SEND CQ event : %d of %d %s\n", i, num_send, ibv_wc_status_str(wc[i].status));
       ibv_get_cq_event(cm_id->send_cq_channel, &cq, &context);

Quoted 2 lines

     //expensive call, ack all received events together
     ibv_ack_cq_events(cm_id->send_cq, num_send);

Quoted 1 line

     ibv_req_notify_cq(cm_id->send_cq, 0);

Quoted 6 lines

   //if the receive completed, prepare to receive more
   if(fds[POLL_RECV_CQ].revents & POLLIN) {
     struct ibv_cq *cq;
     struct ibv_wc wc[10];
     void *context;
     int num_recv=ibv_poll_cq(cm_id->recv_cq, 10, &wc[0]);

Quoted 3 lines · Jonathan Rosser

I have a client and server test program to explore fully asynchronous communication written as close to a conventional sockets application as possible and am encountering difficulty.

Broadly it looks to me like your actions are in the wrong order. A poll based RDMA loop should look like this:

- exit poll
- Check poll bit
- call ibv_get_cq_event
- call ibv_req_notify_cq
- repeatedly call ibv_poll_cq (while rc == num requested)
- Issue new work
- return to poll

Generally, for your own sanity, I recommend splitting into 3 functions
- Do the stuff with ibv_get_cq_event
- Drain and process WC's
- Issue new work

Most real use cases will also want to call the latter two functions from other waiters in the poll loop (ie whatever your wak_fds is for).

Some random mild comments for you:

Quoted 6 lines

Quoted 6 lines

const int NUM_FDS = 4;

const int POLL_CM = 0;
const int POLL_RECV_CQ = 1;
const int POLL_SEND_CQ = 2;
const int POLL_WAKE = 3;

You can use an enum for these constants

Quoted 2 lines

Quoted 2 lines

//prime notification of events on the recv completion queue ibv_req_notify_cq(cm_id->recv_cq, 0);

Do this earlier, before posting recvs, otherwise you could race getting your first recv.

Quoted 9 lines

Quoted 9 lines

while(ret == 0)

 {
   memset(fds, 0, sizeof(pollfd) * NUM_FDS);
   fds[POLL_CM].fd = cm_channel->fd;
   fds[POLL_CM].events = POLLIN;

   fds[POLL_RECV_CQ].fd = cm_id->recv_cq_channel->fd;
   fds[POLL_RECV_CQ].events = POLLIN;

Quoted 5 lines

Quoted 5 lines

   fds[POLL_SEND_CQ].fd = cm_id->send_cq_channel->fd;
   fds[POLL_SEND_CQ].events = POLLIN;

   fds[POLL_WAKE].fd = wake_fds[0];
   fds[POLL_WAKE].events = POLLIN;

The efficient use of poll does not put these inside the main loop. You only need to initialize fd and events once at the start.

Quoted 7 lines

Quoted 7 lines

   if(fds[POLL_CM].revents & POLLIN) {
     struct rdma_cm_event *cm_event;
     ret = rdma_get_cm_event(cm_channel, &cm_event);
     if(ret) {
       perror("client connection rdma_get_cm_event");
     }
     fprintf(stderr, "Got cm event %s

Quoted 4 lines

Quoted 4 lines

     if(cm_event->event == RDMA_CM_EVENT_ESTABLISHED) {
       //send as soon as we are connected
       ibv_req_notify_cq(cm_id->send_cq, 0);

Again, this should be done once, right after the cq is created.

Quoted 7 lines

Quoted 7 lines

   //if the send completed
   if(fds[POLL_SEND_CQ].revents & POLLIN) {
     struct ibv_cq *cq;
     struct ibv_wc wc[10];
     void *context;
     int num_send = ibv_poll_cq(cm_id->send_cq, 10, &wc[0]);
     if(num_send == 0) fprintf(stderr, ".");

Check that num_sends == 10 and loop again

Quoted 2 lines

Quoted 2 lines

for(int i=0; i<num_send; i++) {
fprintf(stderr,"Got SEND CQ event : %d of %d %s

Quoted 1 line

Quoted 1 line

       ibv_get_cq_event(cm_id->send_cq_channel, &cq, &context);

cq_events are not tied to send WC's, this should be done exactly once, prior to calling ibv_poll_cq

Quoted 2 lines

Quoted 2 lines

     //expensive call, ack all received events together
     ibv_ack_cq_events(cm_id->send_cq, num_send);

You don't have to do this at all in the loop unless you are doing multithreaded things. Using num_send is wrong, I use this:

bool checkCQPoll(struct pollfd &p)

{
    if ((p.revents & POLLIN) == 0 ||
        ibv_get_cq_event(comp,&jnk1,&jnk2) != 0)
        return false;

    compEvents++;
    if (compEvents >= INT_MAX)
    {
        ibv_ack_cq_events(cq,compEvents);
        compEvents = 0;
    }
    int rc;
    if ((rc = ibv_req_notify_cq(cq,0)) == -1)
    {
        errno = rc;
	[..]

And then call ibv_ack_cq_events(cq,compEvents) before trying to destroy the CQ. All it used for it synchronizing exits between threads.

Quoted 1 line

Quoted 1 line

     ibv_req_notify_cq(cm_id->send_cq, 0);

Do right after calling ibv_get_cq_event

Quoted 6 lines

Quoted 6 lines

   //if the receive completed, prepare to receive more
   if(fds[POLL_RECV_CQ].revents & POLLIN) {
     struct ibv_cq *cq;
     struct ibv_wc wc[10];
     void *context;
     int num_recv=ibv_poll_cq(cm_id->recv_cq, 10, &wc[0]);

Same problems as for send, they should be the same. Implement a function like my checkCQPoll example and call it for both cases.

Continually posting sends and recvs will get you into trouble, you will run out of recvs and get RNR's. These days the wisdom for implementing RDMA is that you should have explicit message flow control. Ie for something simple like this you could say that getting a recv means another send is OK, but you still need a mechanism to wait for a send buffer to be returned on the send CQ - there is no ordering guarantee.

Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

Hi Jonathan,

I wrote down a test (latency and transfer speed) with RDMA. Server and client work with the same code and they change defined size buffers for n times (loop). In the makefile.txt, you can find an help to use the code.

I tested Intel NetEffect NE020 E10G81GP cards with this code and I found minimum latency about 11 us,maximum transfer speed about 9,6 GBytes, CPU usage up to 90% on client side.
The last value is not good for us.

Quoted 9 lines · Andrea Gozzelino

Hi Jonathan,

I wrote down a test (latency and transfer speed) with RDMA. Server and client work with the same code and they change defined size
buffers for n times (loop). In the makefile.txt, you can find an help to
use the code.

Quoted 6 lines

I tested Intel NetEffect NE020 E10G81GP cards with this code and I found
minimum latency about 11 us,maximum transfer speed about 9,6 GBytes, CPU
usage up to 90% on client side.
The last value is not good for us.

Hi Andrea,

Thanks for the code. With the advice from Jason I have changed my test program to get reliable communication using 1Mbyte buffers. The CPU usage is less than 2% on both client and server for 10Gb throughput. I have Chelsio S310CR.

I find using the poll() approach more natural as I have experience with
conventional sockets based programming before.

The rdma_client/rdma_server example programs from librdmacm were the easiest to start from and I have incrementally changed them from synchronous to asynchronous operation, and moved the internals of the high level functions in <rdma/rdma_verbs.h> into my own code piece by piece. The learning curve is very steep :)

I found this paper quite interesting www.systems.ethz.ch/research/awards/minimizingthehidden.pdf

Cheers,
Jonathan.

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

Broadly it looks to me like your actions are in the wrong order. A poll based RDMA loop should look like this:

- exit poll
- Check poll bit
- call ibv_get_cq_event
- call ibv_req_notify_cq
- repeatedly call ibv_poll_cq (while rc == num requested)
- Issue new work
- return to poll

Generally, for your own sanity, I recommend splitting into 3 functions
- Do the stuff with ibv_get_cq_event
- Drain and process WC's
- Issue new work

doing multithreaded things. Using num_send is wrong, I use this:

bool checkCQPoll(struct pollfd&p)

Continually posting sends and recvs will get you into trouble, you will run out of recvs and get RNR's. These days the wisdom for implementing RDMA is that you should have explicit message flow

control. Ie for something simple like this you could say that getting a recv means another send is OK, but you still need a mechanism to wait for a send buffer to be returned on the send CQ - there is no ordering guarantee.

Could I get some clarification on where there is no ordering guarantee? The WC's do not necessarily come back in the order that the sends were posted?

Quoted 3 lines

Could I get some clarification on where there is no ordering guarantee? The WC's do not necessarily come back in the order that the sends were posted?

For a given queue, completions are always returned in the order that work requests were posted. However there is no ordering between different queues.

Quoted 3 lines

Continually posting sends and recvs will get you into trouble, you will run out of recvs and get RNR's. These days the wisdom for implementing RDMA is that you should have explicit message flow

OK - I appreciate that a real world protocol ought to have flow control rather than just send as fast as possible. I've been trying to exercise the interfaces as far as possible and make sure my RDMA implementation is solid before building something real on top of it.