Add support and tests for AF_IB
cover Add support and tests for AF_IB
From: Sean Hefty <[email protected]>
Add support for AF_IB to rsockets and enhance the librdmacm samples to verify AF_IB operation.
Include one fix for rsetsockopt to return 0 on success when setting SOL_RDMA.
Sean Hefty (5):
rsockets: Support native IB addressing on connected rsockets
examples: Add support for native IB addressing to samples
rsockets: Add ability to set the IB route directly
rsocket: Return 0 on success for SOL_RDMA options
rstream: Use rsocket option to set route directly
examples/rstream.c | 75 +++++++++++++++++++++++++++++++++++++++-------- examples/udaddy.c | 49 +++++++++++++++++++++--------- include/rdma/rsocket.h | 3 +- src/acm.c | 12 ------- src/cma.h | 12 +++++++ src/rsocket.c | 73 ++++++++++++++++++++++++++++++++++++++-------- 6 files changed, 170 insertions(+), 54 deletions(-)
Signature · 6 lines
1.7.3 -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
A message that is not in the archive (sent off-list or not yet received): 1375482022-5953-1-git-send-email-sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org
PATCH 1/5 rsockets: Support native IB addressing on connected rsockets
From: Sean Hefty <[email protected]>
Update rsockets to support AF_IB addresses on connected rsockets. Support for datagram rsockets is more difficult as a result of using real UDP sockets for QP resolution, so that support is deferred. For connected sockets, we need to update internal checks to handle AF_IB.
--- src/acm.c | 12 ------------ src/cma.h | 12 ++++++++++++ src/rsocket.c | 39 +++++++++++++++++++++++++++++---------- 3 files changed, 41 insertions(+), 22 deletions(-)
src/acm.c +0 -12
@@ -116,18 +116,6 @@ static pthread_mutex_t acm_lock = PTHREAD_MUTEX_INITIALIZER;static int sock =-1 ;static uint16_t server_port; -struct ib_connect_hdr { -uint8_t cma_version; -uint8_t ip_version;/* IP version: 7:4 */ -uint16_t port; -uint32_t src_addr[4 ]; -uint32_t dst_addr[4 ]; -#define cma_src_ip4 src_addr[3] -#define cma_src_ip6 src_addr[0] -#define cma_dst_ip4 dst_addr[3] -#define cma_dst_ip6 dst_addr[0] -}; -static int ucma_set_server_port (void ) {FILE *f;
src/cma.h +12 -0
@@ -206,6 +206,18 @@ struct ibv_path_data };#endif +struct ib_connect_hdr { +uint8_t cma_version; +uint8_t ip_version;/* IP version: 7:4 */ +uint16_t port; +uint32_t src_addr[4 ]; +uint32_t dst_addr[4 ]; +#define cma_src_ip4 src_addr[3] +#define cma_src_ip6 src_addr[0] +#define cma_dst_ip4 dst_addr[3] +#define cma_dst_ip6 dst_addr[0] +}; +#ifndef SYSCONFDIR #define SYSCONFDIR "/etc"#endif
src/rsocket.c +29 -10
@@ -187,6 +187,16 @@ struct rs_conn_data {struct rs_sge data_buf; }; +struct rs_conn_private_data { +union { +struct rs_conn_data conn_data; +struct { +struct ib_connect_hdr ib_hdr; +struct rs_conn_data conn_data; + } af_ib; + }; +}; +/* * rsocket states are ordered as passive, connecting, connected, disconnected. */
@@ -965,8 +975,13 @@ static void rs_free(struct rsocket *rs)free (rs); } -static void rs_set_conn_data (struct rsocket *rs,struct rdma_conn_param *param, -struct rs_conn_data *conn) +static size_t rs_conn_data_offset (struct rsocket *rs) +{ +return (rs->cm_id->route.addr.src_addr.sa_family ==AF_IB ) ? +sizeof (struct ib_connect_hdr ) :0 ; +} + +static void rs_format_conn_data (struct rsocket *rs,struct rs_conn_data *conn) { conn->version =1 ; conn->flags = RS_CONN_FLAG_IOMAP |
@@ -982,9 +997,6 @@ static void rs_set_conn_data(struct rsocket *rs, struct rdma_conn_param *param, conn->data_buf.addr =htonll ((uintptr_t ) rs->rbuf); conn->data_buf.length =htonl (rs->rbuf_size >>1 ); conn->data_buf.key =htonl (rs->rmr->rkey); - - param->private_data = conn; - param->private_data_len =sizeof *conn; }static void rs_save_conn_data (struct rsocket *rs,struct rs_conn_data *conn)
@@ -1063,7 +1075,7 @@ int rsocket(int domain, int type, int protocol)struct rsocket *rs;int index, ret; -if ((domain !=PF_INET && domain !=PF_INET6 ) || +if ((domain !=AF_INET && domain !=AF_INET6 && domain !=AF_IB ) || ((type !=SOCK_STREAM ) && (type !=SOCK_DGRAM )) || (type ==SOCK_STREAM && protocol && protocol !=IPPROTO_TCP ) || (type ==SOCK_DGRAM && protocol && protocol !=IPPROTO_UDP ))
@@ -1165,7 +1177,8 @@ int raccept(int socket, struct sockaddr *addr, socklen_t *addrlen)if (ret <0 ) goto err; - creq = (struct rs_conn_data *) new_rs->cm_id->event->param.conn.private_data; + creq = (struct rs_conn_data *) + (new_rs->cm_id->event->param.conn.private_data +rs_conn_data_offset (rs));if (creq->version !=1 ) { ret =ERR (ENOTSUP ); goto err;
@@ -1180,7 +1193,9 @@ int raccept(int socket, struct sockaddr *addr, socklen_t *addrlen)rs_save_conn_data (new_rs, creq); param = new_rs->cm_id->event->param.conn; -rs_set_conn_data (new_rs, ¶m, &cresp); +rs_format_conn_data (new_rs, &cresp); + param.private_data = &cresp; + param.private_data_len =sizeof cresp; ret =rdma_accept (new_rs->cm_id, ¶m);if (!ret) new_rs->state = rs_connect_rdwr;
@@ -1201,7 +1216,8 @@ err:static int rs_do_connect (struct rsocket *rs) {struct rdma_conn_param param; -struct rs_conn_data creq, *cresp; +struct rs_conn_private_data cdata; +struct rs_conn_data *creq, *cresp;int to, ret;switch (rs->state) {
@@ -1246,7 +1262,10 @@ do_connect:break ;memset (¶m,0 ,sizeof param); -rs_set_conn_data (rs, ¶m, &creq); + creq = (void *) &cdata +rs_conn_data_offset (rs); +rs_format_conn_data (rs, creq); + param.private_data = (void *) creq -rs_conn_data_offset (rs); + param.private_data_len =sizeof (*creq) +rs_conn_data_offset (rs); param.flow_control =1 ; param.retry_count =7 ; param.rnr_retry_count =7 ;
Signature · 6 lines
1.7.3 -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 2/5 examples: Add support for native IB addressing to samples
In reply to a message not in the archiveFrom: Sean Hefty <[email protected]>
Allow the user to specify GID addresses (AF_IB) into udaddy and rstream.
--- examples/rstream.c | 62 +++++++++++++++++++++++++++++++++++++++++----------- examples/udaddy.c | 49 ++++++++++++++++++++++++++++------------ 2 files changed, 83 insertions(+), 28 deletions(-)
examples/rstream.c +49 -13
@@ -93,6 +93,8 @@ static char *dst_addr;static char *src_addr;static struct timeval start, end;static void *buf; +static struct rdma_addrinfo rai_hints; +static struct addrinfo ai_hints;static void show_perf (void ) {
@@ -285,18 +287,25 @@ static void set_options(int rs)static int server_listen (void ) { -struct addrinfo hints, *res; +struct rdma_addrinfo *rai; +struct addrinfo *ai;int val, ret; -memset (&hints,0 ,sizeof hints); - hints.ai_flags =AI_PASSIVE ; - ret =getaddrinfo (src_addr, port, &hints, &res); +if (rai_hints.ai_flags) { + rai_hints.ai_flags |=RAI_PASSIVE ; + ret =rdma_getaddrinfo (src_addr, port, &rai_hints, &rai); + }else { + ai_hints.ai_flags |=AI_PASSIVE ; + ret =getaddrinfo (src_addr, port, &ai_hints, &ai); + }if (ret) {perror ("getaddrinfo" );return ret; } - lrs =rs_socket (res->ai_family, res->ai_socktype, res->ai_protocol); + lrs = rai_hints.ai_flags ? +rs_socket (rai->ai_family,SOCK_STREAM ,0 ) : +rs_socket (ai->ai_family,SOCK_STREAM ,0 );if (lrs <0 ) {perror ("rsocket" ); ret = lrs;
@@ -310,7 +319,9 @@ static int server_listen(void) goto close; } - ret =rs_bind (lrs, res->ai_addr, res->ai_addrlen); + ret = rai_hints.ai_flags ? +rs_bind (lrs, rai->ai_src_addr, rai->ai_src_len) : +rs_bind (lrs, ai->ai_addr, ai->ai_addrlen);if (ret) {perror ("rbind" ); goto close;
@@ -324,7 +335,10 @@ close:if (ret)rs_close (lrs); free: -freeaddrinfo (res); +if (rai_hints.ai_flags) +rdma_freeaddrinfo (rai); +else +freeaddrinfo (ai);return ret; }
@@ -362,18 +376,23 @@ static int server_connect(void)static int client_connect (void ) { -struct addrinfo *res; +struct rdma_addrinfo *rai; +struct addrinfo *ai;struct pollfd fds;int ret, err;socklen_t len; - ret =getaddrinfo (dst_addr, port,NULL , &res); + ret = rai_hints.ai_flags ? +rdma_getaddrinfo (dst_addr, port, &rai_hints, &rai) : +getaddrinfo (dst_addr, port, &ai_hints, &ai);if (ret) {perror ("getaddrinfo" );return ret; } - rs =rs_socket (res->ai_family, res->ai_socktype, res->ai_protocol); + rs = rai_hints.ai_flags ? +rs_socket (rai->ai_family,SOCK_STREAM ,0 ) : +rs_socket (ai->ai_family,SOCK_STREAM ,0 );if (rs <0 ) {perror ("rsocket" ); ret = rs;
@@ -383,7 +402,9 @@ static int client_connect(void)set_options (rs);/* TODO: bind client to src_addr */ - ret =rs_connect (rs, res->ai_addr, res->ai_addrlen); + ret = rai_hints.ai_flags ? +rs_connect (rs, rai->ai_dst_addr, rai->ai_dst_len) : +rs_connect (rs, ai->ai_addr, ai->ai_addrlen);if (ret && (errno !=EINPROGRESS )) {perror ("rconnect" ); goto close;
@@ -411,7 +432,10 @@ close:if (ret)rs_close (rs); free: -freeaddrinfo (res); +if (rai_hints.ai_flags) +rdma_freeaddrinfo (rai); +else +freeaddrinfo (ai);return ret; }
@@ -534,7 +558,9 @@ int main(int argc, char **argv) {int op, ret; -while ((op =getopt (argc, argv,"s:b:B:I:C:S:p:T:" )) !=-1 ) { + ai_hints.ai_socktype =SOCK_STREAM ; + rai_hints.ai_port_space =RDMA_PS_TCP ; +while ((op =getopt (argc, argv,"s:b:f:B:I:C:S:p:T:" )) !=-1 ) {switch (op) {case 's' : dst_addr = optarg;
@@ -542,6 +568,14 @@ int main(int argc, char **argv)case 'b' : src_addr = optarg;break ; +case 'f' : +if (!strncasecmp ("ip" , optarg,2 )) { + ai_hints.ai_flags =AI_NUMERICHOST ; + }else if (!strncasecmp ("gid" , optarg,3 )) { + rai_hints.ai_flags =RAI_NUMERICHOST |RAI_FAMILY ; + rai_hints.ai_family =AF_IB ; + } +break ;case 'B' : buffer_size =atoi (optarg);break ;
@@ -572,6 +606,8 @@ int main(int argc, char **argv)printf ("usage: %s\n" , argv[0 ]);printf ("\t[-s server_address]\n" );printf ("\t[-b bind_address]\n" ); +printf ("\t[-f address_format]\n" ); +printf ("\t name, ip, ipv6, or gid\n" );printf ("\t[-B buffer_size]\n" );printf ("\t[-I iterations]\n" );printf ("\t[-C transfer_count]\n" );
examples/udaddy.c +34 -15
110 lines not on this page · raw message
Signature · 6 lines
1.7.3 -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 3/5 rsockets: Add ability to set the IB route directly
In reply to a message not in the archiveFrom: Sean Hefty <[email protected]>
Add an RDMA specific rsocket option that allows the user to program the RDMA route directly. This is useful for apps that have path record data available, e.g. from ibacm.
--- include/rdma/rsocket.h | 3 ++- src/rsocket.c | 30 +++++++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 4 deletions(-)
include/rdma/rsocket.h +2 -1
@@ -78,7 +78,8 @@ enum {RDMA_SQSIZE ,RDMA_RQSIZE ,RDMA_INLINE , -RDMA_IOMAPSIZE +RDMA_IOMAPSIZE , +RDMA_ROUTE }; int rsetsockopt(int socket, int level, int optname,
src/rsocket.c +27 -3
@@ -329,6 +329,8 @@ struct rsocket {long fd_flags;uint64_t so_opts;uint64_t ipv6_opts; +void *optval; +size_t optlen;int state;int cq_armed;int retries;
@@ -1243,13 +1245,26 @@ resolve_addr: rs->retries =0 ; resolve_route: to =1000 << rs->retries++; - ret =rdma_resolve_route (rs->cm_id, to); -if (!ret) - goto do_connect; +if (rs->optval) { + ret =rdma_set_option (rs->cm_id,RDMA_OPTION_IB , +RDMA_OPTION_IB_PATH , rs->optval, + rs->optlen); +free (rs->optval); + rs->optval =NULL ; +if (!ret) { + rs->state = rs_resolving_route; + goto resolving_route; + } + }else { + ret =rdma_resolve_route (rs->cm_id, to); +if (!ret) + goto do_connect; + }if (errno ==EAGAIN || errno ==EWOULDBLOCK ) rs->state = rs_resolving_route;break ;case rs_resolving_route: +resolving_route: ret =ucma_complete (rs->cm_id);if (ret) {if (errno ==ETIMEDOUT && rs->retries <=RS_CONN_RETRIES )
@@ -3312,6 +3327,15 @@ int rsetsockopt(int socket, int level, int optname, rs->target_iomap_size = (uint16_t )rs_scale_to_value ( (uint8_t )rs_value_to_scale (*(int *) optval,8 ),8 );break ; +case RDMA_ROUTE : +if ((rs->optval =calloc (optlen,1 ))) { +memcpy (rs->optval, optval, optlen); + rs->optlen = optlen; + ret =0 ; + }else { + ret =ERR (ENOMEM ); + } +break ;default :break ; }
Signature · 6 lines
1.7.3 -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 4/5 rsocket: Return 0 on success for SOL_RDMA options
In reply to a message not in the archiveFrom: Sean Hefty <[email protected]>
The processing of SOL_RDMA does not set the return value in the case of successfully handled options.
--- src/rsocket.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-)
src/rsocket.c +4 -0
@@ -3314,18 +3314,22 @@ int rsetsockopt(int socket, int level, int optname,switch (optname) {case RDMA_SQSIZE : rs->sq_size =min ((*(uint32_t *) optval),RS_QP_MAX_SIZE ); + ret =0 ;break ;case RDMA_RQSIZE : rs->rq_size =min ((*(uint32_t *) optval),RS_QP_MAX_SIZE ); + ret =0 ;break ;case RDMA_INLINE : rs->sq_inline =min (*(uint32_t *) optval,RS_QP_MAX_SIZE );if (rs->sq_inline <RS_MIN_INLINE ) rs->sq_inline =RS_MIN_INLINE ; + ret =0 ;break ;case RDMA_IOMAPSIZE : rs->target_iomap_size = (uint16_t )rs_scale_to_value ( (uint8_t )rs_value_to_scale (*(int *) optval,8 ),8 ); + ret =0 ;break ;case RDMA_ROUTE :if ((rs->optval =calloc (optlen,1 ))) {
Signature · 6 lines
1.7.3 -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 5/5 rstream: Use rsocket option to set route directly
In reply to a message not in the archiveFrom: Sean Hefty <[email protected]>
If we're using GID addressing, rdma_getaddrinfo can return routing data directly. Add an option for the user to indicate that rdma_getaddrinfo should be called in place of getaddrinfo. And if routing data is available, call rsetsockopt to set the route.
This helps test rsockets when ibacm and AF_IB support are available.
--- examples/rstream.c | 53 ++++++++++++++++++++++++++++++++------------------- 1 files changed, 33 insertions(+), 20 deletions(-)
examples/rstream.c +33 -20
@@ -75,6 +75,7 @@ static struct test_size_param test_size[] = {static int rs, lrs;static int use_async; +static int use_rgai;static int verify;static int flags =MSG_DONTWAIT ;static int poll_timeout =0 ;
@@ -287,11 +288,11 @@ static void set_options(int rs)static int server_listen (void ) { -struct rdma_addrinfo *rai; +struct rdma_addrinfo *rai =NULL ;struct addrinfo *ai;int val, ret; -if (rai_hints.ai_flags) { +if (use_rgai) { rai_hints.ai_flags |=RAI_PASSIVE ; ret =rdma_getaddrinfo (src_addr, port, &rai_hints, &rai); }else {
@@ -303,9 +304,8 @@ static int server_listen(void)return ret; } - lrs = rai_hints.ai_flags ? -rs_socket (rai->ai_family,SOCK_STREAM ,0 ) : -rs_socket (ai->ai_family,SOCK_STREAM ,0 ); + lrs = rai ?rs_socket (rai->ai_family,SOCK_STREAM ,0 ) : +rs_socket (ai->ai_family,SOCK_STREAM ,0 );if (lrs <0 ) {perror ("rsocket" ); ret = lrs;
@@ -319,9 +319,8 @@ static int server_listen(void) goto close; } - ret = rai_hints.ai_flags ? -rs_bind (lrs, rai->ai_src_addr, rai->ai_src_len) : -rs_bind (lrs, ai->ai_addr, ai->ai_addrlen); + ret = rai ?rs_bind (lrs, rai->ai_src_addr, rai->ai_src_len) : +rs_bind (lrs, ai->ai_addr, ai->ai_addrlen);if (ret) {perror ("rbind" ); goto close;
@@ -335,7 +334,7 @@ close:if (ret)rs_close (lrs); free: -if (rai_hints.ai_flags) +if (rai)rdma_freeaddrinfo (rai);else freeaddrinfo (ai);
@@ -376,23 +375,22 @@ static int server_connect(void)static int client_connect (void ) { -struct rdma_addrinfo *rai; +struct rdma_addrinfo *rai =NULL ;struct addrinfo *ai;struct pollfd fds;int ret, err;socklen_t len; - ret = rai_hints.ai_flags ? -rdma_getaddrinfo (dst_addr, port, &rai_hints, &rai) : -getaddrinfo (dst_addr, port, &ai_hints, &ai); + ret = use_rgai ?rdma_getaddrinfo (dst_addr, port, &rai_hints, &rai) : +getaddrinfo (dst_addr, port, &ai_hints, &ai); +if (ret) {perror ("getaddrinfo" );return ret; } - rs = rai_hints.ai_flags ? -rs_socket (rai->ai_family,SOCK_STREAM ,0 ) : -rs_socket (ai->ai_family,SOCK_STREAM ,0 ); + rs = rai ?rs_socket (rai->ai_family,SOCK_STREAM ,0 ) : +rs_socket (ai->ai_family,SOCK_STREAM ,0 );if (rs <0 ) {perror ("rsocket" ); ret = rs;
@@ -402,9 +400,17 @@ static int client_connect(void)set_options (rs);/* TODO: bind client to src_addr */ - ret = rai_hints.ai_flags ? -rs_connect (rs, rai->ai_dst_addr, rai->ai_dst_len) : -rs_connect (rs, ai->ai_addr, ai->ai_addrlen); +if (rai && rai->ai_route) { + ret =rs_setsockopt (rs,SOL_RDMA ,RDMA_ROUTE , rai->ai_route, + rai->ai_route_len); +if (ret) { +perror ("rsetsockopt RDMA_ROUTE" ); + goto close; + } + } + + ret = rai ?rs_connect (rs, rai->ai_dst_addr, rai->ai_dst_len) : +rs_connect (rs, ai->ai_addr, ai->ai_addrlen);if (ret && (errno !=EINPROGRESS )) {perror ("rconnect" ); goto close;
@@ -432,7 +438,7 @@ close:if (ret)rs_close (rs); free: -if (rai_hints.ai_flags) +if (rai)rdma_freeaddrinfo (rai);else freeaddrinfo (ai);
@@ -527,6 +533,9 @@ static int set_test_opt(char *optarg)case 'n' : flags |=MSG_DONTWAIT ;break ; +case 'r' : + use_rgai =1 ; +break ;case 'v' : verify =1 ;break ;
@@ -542,6 +551,8 @@ static int set_test_opt(char *optarg) flags = (flags & ~MSG_DONTWAIT ) |MSG_WAITALL ; }else if (!strncasecmp ("nonblock" , optarg,8 )) { flags |=MSG_DONTWAIT ; + }else if (strncasecmp ("resolve" , optarg,7 )) { + use_rgai =1 ; }else if (!strncasecmp ("verify" , optarg,6 )) { verify =1 ; }else if (!strncasecmp ("fork" , optarg,4 )) {
@@ -574,6 +585,7 @@ int main(int argc, char **argv) }else if (!strncasecmp ("gid" , optarg,3 )) { rai_hints.ai_flags =RAI_NUMERICHOST |RAI_FAMILY ; rai_hints.ai_family =AF_IB ; + use_rgai =1 ; }break ;case 'B' :
@@ -619,6 +631,7 @@ int main(int argc, char **argv)printf ("\t b|blocking - use blocking calls\n" );printf ("\t f|fork - fork server processing\n" );printf ("\t n|nonblocking - use nonblocking calls\n" ); +printf ("\t r|resolve - use rdma cm to resolve address\n" );printf ("\t v|verify - verify data\n" );exit (1 ); }
Signature · 6 lines
1.7.3 -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html