asynchronous operation with poll()

DISCUSSION Jonathan Rosser 4 participants 9 messages Nov 9, 2010 – Nov 11, 2010 3 messages not in the archive InfiniBand

lore

Share & tools

lore

Jonathan Rosser

author

2010-11-09 15:58 UTC #0 lore raw

I have a client and server test program to explore fully asynchronous communication written as close to a conventional sockets application as possible and am encountering difficulty.

Both programs run the same code in a thread, sending buffers to each other as fast as possible. On the client side only, my poll() call never blocks and cm_id->send_cq_channel->fd always seems to be readable. This causes the program to loop wildly and consume 100% CPU.

Any ideas? I have ensured that O_NONBLOCK is set on the underlying file descriptors. I'm not sure why the server side should run with almost no cpu usage yet the client does not.

Here is the client/server loop:

Quoted 169 lines

  struct ibv_mr *mr;
  int ret;
  int send_buf_num = 0;
  int recv_buf_num = 0;

  #define NUM_BUFFERS 20
  #define SIZE 1024*1024
  uint8_t *buffer = (uint8_t*)malloc(SIZE * NUM_BUFFERS * 2);
  uint8_t *send_msg[NUM_BUFFERS];
  uint8_t *recv_msg[NUM_BUFFERS];

  for(int i=0; i<NUM_BUFFERS; i++) {
    send_msg[i] = buffer + (i*SIZE);
    recv_msg[i] = buffer + ((i+NUM_BUFFERS) * SIZE);
  }

  //--------------------------------------------------------------------
  // setup
  fprintf(stderr, "rdma_reg_msgs\n");
  mr = rdma_reg_msgs(cm_id, buffer, SIZE*NUM_BUFFERS*2);
  if (!mr) {
    perror("rdma_reg_msgs");
  }

  //prepare to for the first receive before connecting
  for(int i=0; i<10; i++) {
    fprintf(stderr, "rdma_post_recv\n");
    ret = rdma_post_recv(cm_id, NULL, recv_msg[recv_buf_num++], SIZE, mr);
    recv_buf_num %= NUM_BUFFERS;
    if (ret) {
      perror("rdma_post_recv");
    }
  }

  //connect
  fprintf(stderr, "rdma_connect\n");
  ret = rdma_connect(cm_id, NULL);
  if (ret) {
    perror("rdma_connect");
  }

  const int NUM_FDS = 4;

  const int POLL_CM = 0;
  const int POLL_RECV_CQ = 1;
  const int POLL_SEND_CQ = 2;
  const int POLL_WAKE = 3;
  struct pollfd fds[NUM_FDS];

//prime notification of events on the recv completion queue ibv_req_notify_cq(cm_id->recv_cq, 0);
//

  //--------------------------------------------------------------------
  // main loop
  while(ret == 0)
  {
    memset(fds, 0, sizeof(pollfd) * NUM_FDS);
    fds[POLL_CM].fd = cm_channel->fd;
    fds[POLL_CM].events = POLLIN;

    fds[POLL_RECV_CQ].fd = cm_id->recv_cq_channel->fd;
    fds[POLL_RECV_CQ].events = POLLIN;

    fds[POLL_SEND_CQ].fd = cm_id->send_cq_channel->fd;
    fds[POLL_SEND_CQ].events = POLLIN;

    fds[POLL_WAKE].fd = wake_fds[0];
    fds[POLL_WAKE].events = POLLIN;

    int nready = poll(fds, NUM_FDS, -1);
    if(nready < 0) {
      perror("poll");
    }

    if(fds[POLL_CM].revents & POLLIN) {
      struct rdma_cm_event *cm_event;
      ret = rdma_get_cm_event(cm_channel, &cm_event);
      if(ret) {
        perror("client connection rdma_get_cm_event");
      }
      fprintf(stderr, "Got cm event %s\n", rdma_event_str(cm_event->event));

      if(cm_event->event == RDMA_CM_EVENT_ESTABLISHED) {
        //send as soon as we are connected
        ibv_req_notify_cq(cm_id->send_cq, 0);
        ret = rdma_post_send(cm_id, NULL, send_msg[send_buf_num], SIZE, mr, 0);
        send_buf_num++;
        send_buf_num %= NUM_BUFFERS;
        if (ret) {
          perror("rdma_post_send");
        }
      }

      int finish=0;
      if(cm_event->event == RDMA_CM_EVENT_DISCONNECTED ||
         cm_event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
          finish = 1;

      rdma_ack_cm_event(cm_event);
      if(finish) {
        goto out;
      }
    }

    //if the send completed
    if(fds[POLL_SEND_CQ].revents & POLLIN) {
      struct ibv_cq *cq;
      struct ibv_wc wc[10];
      void *context;
      int num_send = ibv_poll_cq(cm_id->send_cq, 10, &wc[0]);

      if(num_send == 0) fprintf(stderr, ".");

      for(int i=0; i<num_send; i++) {
        fprintf(stderr,"Got SEND CQ event : %d of %d %s\n", i, num_send, ibv_wc_status_str(wc[i].status));
        ibv_get_cq_event(cm_id->send_cq_channel, &cq, &context);
        assert(cq == cm_id->send_cq);

        //our send completed, send some more right away
        fprintf(stderr, "rdma_post_send\n");
        ret = rdma_post_send(cm_id, NULL, send_msg[send_buf_num++], SIZE, mr, 0);
        send_buf_num %= NUM_BUFFERS;
        if (ret) {
          perror("rdma_post_send");
        }
      }

      //expensive call, ack all received events together
      ibv_ack_cq_events(cm_id->send_cq, num_send);
      ibv_req_notify_cq(cm_id->send_cq, 0);
    }

    //if the receive completed, prepare to receive more
    if(fds[POLL_RECV_CQ].revents & POLLIN) {
      struct ibv_cq *cq;
      struct ibv_wc wc[10];
      void *context;
      int num_recv=ibv_poll_cq(cm_id->recv_cq, 10, &wc[0]);

      for(int i=0; i<num_recv; i++) {
        fprintf(stderr,"Got RECV CQ event : %d of %d %s\n", i, num_recv, ibv_wc_status_str(wc[i].status));
        ibv_get_cq_event(cm_id->recv_cq_channel, &cq, &context);
        assert(cq == cm_id->recv_cq);

        //we received some payload, prepare to receive more
        fprintf(stderr, "rdma_post_recv\n");
        ret = rdma_post_recv(cm_id, NULL, recv_msg[recv_buf_num++], SIZE, mr);
        recv_buf_num %= NUM_BUFFERS;
        if (ret) {
           perror("rdma_post_recv");
        }
      }

      //expensive call, ack all received events together
      ibv_ack_cq_events(cm_id->recv_cq, num_recv);
      ibv_req_notify_cq(cm_id->recv_cq, 0);
    }

    if(fds[POLL_WAKE].revents & POLLIN) {
      fprintf(stderr, "poll WAKE\n");
      char buffer[1];
      int nread = read(wake_fds[0], buffer, 1);
      fprintf(stderr, "Got Wake event %d\n", nread);
      goto out;
    }

Quoted 7 lines

out:
rdma_disconnect(cm_id);
rdma_dereg_mr(mr);
rdma_destroy_ep(cm_id);
  free(buffer);
  fprintf(stderr, "poll: client completed\n");

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

Jason Gunthorpe

2010-11-09 20:44 UTC #1 lore raw

Quoted 3 lines · Jonathan Rosser

I have a client and server test program to explore fully asynchronous communication written as close to a conventional sockets application as possible and am encountering difficulty.

Broadly it looks to me like your actions are in the wrong order. A poll based RDMA loop should look like this:

- exit poll
- Check poll bit
- call ibv_get_cq_event
- call ibv_req_notify_cq
- repeatedly call ibv_poll_cq (while rc == num requested)
- Issue new work
- return to poll

Generally, for your own sanity, I recommend splitting into 3 functions
- Do the stuff with ibv_get_cq_event
- Drain and process WC's
- Issue new work

Most real use cases will also want to call the latter two functions from other waiters in the poll loop (ie whatever your wak_fds is for).

Some random mild comments for you:

Quoted 6 lines

Quoted 6 lines
const int NUM_FDS = 4;
const int POLL_CM = 0;
const int POLL_RECV_CQ = 1;
const int POLL_SEND_CQ = 2;
const int POLL_WAKE = 3;

You can use an enum for these constants

Quoted 2 lines

Quoted 2 lines
//prime notification of events on the recv completion queue ibv_req_notify_cq(cm_id->recv_cq, 0);

Do this earlier, before posting recvs, otherwise you could race getting your first recv.

Quoted 6 lines

while(ret == 0)

 {
   memset(fds, 0, sizeof(pollfd) * NUM_FDS);
   fds[POLL_CM].fd = cm_channel->fd;
   fds[POLL_CM].events = POLLIN;

Quoted 8 lines

   fds[POLL_RECV_CQ].fd = cm_id->recv_cq_channel->fd;
   fds[POLL_RECV_CQ].events = POLLIN;

   fds[POLL_SEND_CQ].fd = cm_id->send_cq_channel->fd;
   fds[POLL_SEND_CQ].events = POLLIN;

   fds[POLL_WAKE].fd = wake_fds[0];
   fds[POLL_WAKE].events = POLLIN;

The efficient use of poll does not put these inside the main loop. You only need to initialize fd and events once at the start.

Quoted 11 lines

   if(fds[POLL_CM].revents & POLLIN) {
     struct rdma_cm_event *cm_event;
     ret = rdma_get_cm_event(cm_channel, &cm_event);
     if(ret) {
       perror("client connection rdma_get_cm_event");
     }
     fprintf(stderr, "Got cm event %s\n", rdma_event_str(cm_event->event));

     if(cm_event->event == RDMA_CM_EVENT_ESTABLISHED) {
       //send as soon as we are connected
       ibv_req_notify_cq(cm_id->send_cq, 0);

Again, this should be done once, right after the cq is created.

Quoted 7 lines

   //if the send completed
   if(fds[POLL_SEND_CQ].revents & POLLIN) {
     struct ibv_cq *cq;
     struct ibv_wc wc[10];
     void *context;
     int num_send = ibv_poll_cq(cm_id->send_cq, 10, &wc[0]);
     if(num_send == 0) fprintf(stderr, ".");

Check that num_sends == 10 and loop again

Quoted 3 lines

     for(int i=0; i<num_send; i++) {
       fprintf(stderr,"Got SEND CQ event : %d of %d %s\n", i, num_send, ibv_wc_status_str(wc[i].status));
       ibv_get_cq_event(cm_id->send_cq_channel, &cq, &context);

cq_events are not tied to send WC's, this should be done exactly once, prior to calling ibv_poll_cq

Quoted 2 lines

     //expensive call, ack all received events together
     ibv_ack_cq_events(cm_id->send_cq, num_send);

You don't have to do this at all in the loop unless you are doing multithreaded things. Using num_send is wrong, I use this:

bool checkCQPoll(struct pollfd &p)

{
    if ((p.revents & POLLIN) == 0 ||
        ibv_get_cq_event(comp,&jnk1,&jnk2) != 0)
        return false;

    compEvents++;
    if (compEvents >= INT_MAX)
    {
        ibv_ack_cq_events(cq,compEvents);
        compEvents = 0;
    }
    int rc;
    if ((rc = ibv_req_notify_cq(cq,0)) == -1)
    {
        errno = rc;
	[..]

And then call ibv_ack_cq_events(cq,compEvents) before trying to destroy the CQ. All it used for it synchronizing exits between threads.

Quoted 1 line

     ibv_req_notify_cq(cm_id->send_cq, 0);

Do right after calling ibv_get_cq_event

Quoted 6 lines

   //if the receive completed, prepare to receive more
   if(fds[POLL_RECV_CQ].revents & POLLIN) {
     struct ibv_cq *cq;
     struct ibv_wc wc[10];
     void *context;
     int num_recv=ibv_poll_cq(cm_id->recv_cq, 10, &wc[0]);

Same problems as for send, they should be the same. Implement a function like my checkCQPoll example and call it for both cases.

Continually posting sends and recvs will get you into trouble, you will run out of recvs and get RNR's. These days the wisdom for implementing RDMA is that you should have explicit message flow control. Ie for something simple like this you could say that getting a recv means another send is OK, but you still need a mechanism to wait for a send buffer to be returned on the send CQ - there is no ordering guarantee.

Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): 20101109204452.GG909-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org

Andrea Gozzelino

2010-11-10 10:30 UTC #3 lore raw

Hi Jonathan,

I wrote down a test (latency and transfer speed) with RDMA. Server and client work with the same code and they change defined size buffers for n times (loop). In the makefile.txt, you can find an help to use the code.

I tested Intel NetEffect NE020 E10G81GP cards with this code and I found minimum latency about 11 us,maximum transfer speed about 9,6 GBytes, CPU usage up to 90% on client side.
The last value is not good for us.

I hope these help you.
Thank you,
Andrea

Andrea Gozzelino

INFN - Laboratori Nazionali di Legnaro (LNL)
Viale dell'Universita' 2 -I-35020 - Legnaro (PD)- ITALIA
Office: E-101
Tel: +39 049 8068346
Fax: +39 049 641925
Mail: andrea.gozzelino-PK20h7lG/[email protected]
Cell: +39 3488245552

Quoted 51 lines · Jason Gunthorpe

Quoted 3 lines · Jonathan Rosser
I have a client and server test program to explore fully asynchronous communication written as close to a conventional sockets application as possible and am encountering difficulty.
Broadly it looks to me like your actions are in the wrong order. A poll based RDMA loop should look like this:
- exit poll
- Check poll bit
- call ibv_get_cq_event
- call ibv_req_notify_cq
- repeatedly call ibv_poll_cq (while rc == num requested)
- Issue new work
- return to poll
Generally, for your own sanity, I recommend splitting into 3 functions
- Do the stuff with ibv_get_cq_event
- Drain and process WC's
- Issue new work
Most real use cases will also want to call the latter two functions from other waiters in the poll loop (ie whatever your wak_fds is for).
Some random mild comments for you:
Quoted 6 lines
Quoted 6 lines
const int NUM_FDS = 4;
const int POLL_CM = 0;
const int POLL_RECV_CQ = 1;
const int POLL_SEND_CQ = 2;
const int POLL_WAKE = 3;
You can use an enum for these constants
Quoted 2 lines
Quoted 2 lines
//prime notification of events on the recv completion queue ibv_req_notify_cq(cm_id->recv_cq, 0);
Do this earlier, before posting recvs, otherwise you could race getting your first recv.
Quoted 9 lines
Quoted 9 lines
while(ret == 0)
 {
   memset(fds, 0, sizeof(pollfd) * NUM_FDS);
   fds[POLL_CM].fd = cm_channel->fd;
   fds[POLL_CM].events = POLLIN;

   fds[POLL_RECV_CQ].fd = cm_id->recv_cq_channel->fd;
   fds[POLL_RECV_CQ].events = POLLIN;

Quoted 16 lines

Quoted 5 lines
Quoted 5 lines
   fds[POLL_SEND_CQ].fd = cm_id->send_cq_channel->fd;
   fds[POLL_SEND_CQ].events = POLLIN;

   fds[POLL_WAKE].fd = wake_fds[0];
   fds[POLL_WAKE].events = POLLIN;
The efficient use of poll does not put these inside the main loop. You only need to initialize fd and events once at the start.
Quoted 7 lines
Quoted 7 lines
   if(fds[POLL_CM].revents & POLLIN) {
     struct rdma_cm_event *cm_event;
     ret = rdma_get_cm_event(cm_channel, &cm_event);
     if(ret) {
       perror("client connection rdma_get_cm_event");
     }
     fprintf(stderr, "Got cm event %s

", rdma_event_str(cm_event->event));

Quoted 15 lines

Quoted 4 lines

     if(cm_event->event == RDMA_CM_EVENT_ESTABLISHED) {
       //send as soon as we are connected
       ibv_req_notify_cq(cm_id->send_cq, 0);

Again, this should be done once, right after the cq is created.

Quoted 7 lines

   //if the send completed
   if(fds[POLL_SEND_CQ].revents & POLLIN) {
     struct ibv_cq *cq;
     struct ibv_wc wc[10];
     void *context;
     int num_send = ibv_poll_cq(cm_id->send_cq, 10, &wc[0]);
     if(num_send == 0) fprintf(stderr, ".");

Quoted 4 lines

Check that num_sends == 10 and loop again
Quoted 2 lines
Quoted 2 lines
for(int i=0; i<num_send; i++) {
fprintf(stderr,"Got SEND CQ event : %d of %d %s

", i, num_send, ibv_wc_status_str(wc[i].status));

Quoted 62 lines

Quoted 1 line
Quoted 1 line
       ibv_get_cq_event(cm_id->send_cq_channel, &cq, &context);
cq_events are not tied to send WC's, this should be done exactly once, prior to calling ibv_poll_cq
Quoted 2 lines
Quoted 2 lines
     //expensive call, ack all received events together
     ibv_ack_cq_events(cm_id->send_cq, num_send);
You don't have to do this at all in the loop unless you are doing multithreaded things. Using num_send is wrong, I use this:
bool checkCQPoll(struct pollfd &p)
{
    if ((p.revents & POLLIN) == 0 ||
        ibv_get_cq_event(comp,&jnk1,&jnk2) != 0)
        return false;

    compEvents++;
    if (compEvents >= INT_MAX)
    {
        ibv_ack_cq_events(cq,compEvents);
        compEvents = 0;
    }
    int rc;
    if ((rc = ibv_req_notify_cq(cq,0)) == -1)
    {
        errno = rc;
	[..]
And then call ibv_ack_cq_events(cq,compEvents) before trying to destroy the CQ. All it used for it synchronizing exits between threads.
Quoted 1 line
Quoted 1 line
     ibv_req_notify_cq(cm_id->send_cq, 0);
Do right after calling ibv_get_cq_event
Quoted 6 lines
Quoted 6 lines
   //if the receive completed, prepare to receive more
   if(fds[POLL_RECV_CQ].revents & POLLIN) {
     struct ibv_cq *cq;
     struct ibv_wc wc[10];
     void *context;
     int num_recv=ibv_poll_cq(cm_id->recv_cq, 10, &wc[0]);
Same problems as for send, they should be the same. Implement a function like my checkCQPoll example and call it for both cases.
Continually posting sends and recvs will get you into trouble, you will run out of recvs and get RNR's. These days the wisdom for implementing RDMA is that you should have explicit message flow control. Ie for something simple like this you could say that getting a recv means another send is OK, but you still need a mechanism to wait for a send buffer to be returned on the send CQ - there is no ordering guarantee.
Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): 4538690.1289385043068.SLOX.WebMail.wwwrun-XDIR3SKYeFbgKi2NxijLtw@public.gmane.org

Jonathan Rosser

author

2010-11-10 16:05 UTC #5 lore raw

Quoted 11 lines · Andrea Gozzelino

Hi Jonathan,
I wrote down a test (latency and transfer speed) with RDMA. Server and client work with the same code and they change defined size buffers for n times (loop). In the makefile.txt, you can find an help to use the code.
I tested Intel NetEffect NE020 E10G81GP cards with this code and I found minimum latency about 11 us,maximum transfer speed about 9,6 GBytes, CPU usage up to 90% on client side.
The last value is not good for us.

Hi Andrea,

Thanks for the code. With the advice from Jason I have changed my test program to get reliable communication using 1Mbyte buffers. The CPU usage is less than 2% on both client and server for 10Gb throughput. I have Chelsio S310CR.

I find using the poll() approach more natural as I have experience with conventional sockets based programming before.

The rdma_client/rdma_server example programs from librdmacm were the easiest to start from and I have incrementally changed them from synchronous to asynchronous operation, and moved the internals of the high level functions in <rdma/rdma_verbs.h> into my own code piece by piece. The learning curve is very steep :)

I found this paper quite interesting www.systems.ethz.ch/research/awards/minimizingthehidden.pdf

Cheers,
Jonathan.

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

Andrea Gozzelino

2010-11-11 08:43 UTC #6 lore raw

Quoted 41 lines · Jonathan Rosser

Quoted 9 lines · Andrea Gozzelino
Hi Jonathan,
I wrote down a test (latency and transfer speed) with RDMA. Server and client work with the same code and they change defined size
buffers for n times (loop). In the makefile.txt, you can find an help to
use the code.
Quoted 6 lines
I tested Intel NetEffect NE020 E10G81GP cards with this code and I found
minimum latency about 11 us,maximum transfer speed about 9,6 GBytes, CPU
usage up to 90% on client side.
The last value is not good for us.
Hi Andrea,
Thanks for the code. With the advice from Jason I have changed my test program to get reliable communication using 1Mbyte buffers. The CPU usage is less than 2% on both client and server for 10Gb throughput. I have Chelsio S310CR.
I find using the poll() approach more natural as I have experience with
conventional sockets based programming before.
The rdma_client/rdma_server example programs from librdmacm were the easiest to start from and I have incrementally changed them from synchronous to asynchronous operation, and moved the internals of the high level functions in <rdma/rdma_verbs.h> into my own code piece by piece. The learning curve is very steep :)
I found this paper quite interesting www.systems.ethz.ch/research/awards/minimizingthehidden.pdf
Cheers,
Jonathan.

Quoted 6 lines

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

Hi Jonathan,

I would like testing my Intel NetEffect cards with your code, if it is possible.
Could you please send me the files client/server and the compiling commands?

Thank you very much for the pdf paper: I'm going to read it during the morning.

Cheers,
Andrea Gozzelino

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

Jonathan Rosser

author

2010-11-10 14:39 UTC #7 lore raw

In reply to a message not in the archive

Quoted 11 lines · Jason Gunthorpe

Broadly it looks to me like your actions are in the wrong order. A poll based RDMA loop should look like this:
- exit poll
- Check poll bit
- call ibv_get_cq_event
- call ibv_req_notify_cq
- repeatedly call ibv_poll_cq (while rc == num requested)
- Issue new work
- return to poll

Quoted 5 lines

Generally, for your own sanity, I recommend splitting into 3 functions
- Do the stuff with ibv_get_cq_event
- Drain and process WC's
- Issue new work

Hi Jason,

Thanks very much for your advice. I had misunderstood the relationship between CQ events and available WC's.

Quoted 3 lines

doing multithreaded things. Using num_send is wrong, I use this:
bool checkCQPoll(struct pollfd&p)

Right - using a function like your checkCQPoll has sorted out the behaviour of the poll() loop.

Quoted 3 lines

Continually posting sends and recvs will get you into trouble, you will run out of recvs and get RNR's. These days the wisdom for implementing RDMA is that you should have explicit message flow

OK - I appreciate that a real world protocol ought to have flow control rather than just send as fast as possible. I've been trying to exercise the interfaces as far as possible and make sure my RDMA implementation is solid before building something real on top of it.

Quoted 4 lines

control. Ie for something simple like this you could say that getting a recv means another send is OK, but you still need a mechanism to wait for a send buffer to be returned on the send CQ - there is no ordering guarantee.

Could I get some clarification on where there is no ordering guarantee? The WC's do not necessarily come back in the order that the sends were posted?

Many Thanks,
Jonathan.

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

Roland Dreier

2010-11-10 17:43 UTC #8 lore raw

Quoted 3 lines

Could I get some clarification on where there is no ordering guarantee? The WC's do not necessarily come back in the order that the sends were posted?

For a given queue, completions are always returned in the order that work requests were posted. However there is no ordering between different queues.

- R.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): [email protected]

Jason Gunthorpe

2010-11-10 17:56 UTC #10 lore raw

Quoted 7 lines · Roland Dreier

Quoted 3 lines
Could I get some clarification on where there is no ordering guarantee? The WC's do not necessarily come back in the order that the sends were posted?
For a given queue, completions are always returned in the order that work requests were posted. However there is no ordering between different queues.

Right, so if you design a scheme where getting a recv grants a send credit then you are doing

Post Send #1 ---> Recv WC
Recv WC <---- Post Send #2
Send WC #2 Send WC #2
PostSend #3 ---> Recv WC

and there is no order guarentee for when you get a Recv WC vs a Send WC, even though by protocol design they are in fact ordered. If you load the HCA the Send Complete WC's do get behind Recv WCs.

This is true even if you use the same CQ for Recv WC and Send Complete WCs.

In practice this can make the implementation quite troublesome since the posting of a new Send can't necessarily be done in the Recv WC path as no send buffers could be available at that time.

I've found properly placing all the activities in a RDMA system to be the hardest challenge of the whole design. Avoiding deadlock and starvation and untangling the various data dependencies can be hard :(

Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

Jason Gunthorpe

2010-11-10 18:04 UTC #11 lore raw

In reply to Jonathan Rosser

Quoted 8 lines · Jonathan Rosser

Quoted 3 lines
Continually posting sends and recvs will get you into trouble, you will run out of recvs and get RNR's. These days the wisdom for implementing RDMA is that you should have explicit message flow
OK - I appreciate that a real world protocol ought to have flow control rather than just send as fast as possible. I've been trying to exercise the interfaces as far as possible and make sure my RDMA implementation is solid before building something real on top of it.

With iWarp if you send more than the receiver is able to receive you will break the connection with an error. Some kind of flow control scheme is mandatory..

Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html