Skip to contentSkip to conversation

iser: configurable bufsize and ipv6 support

PATCH Applied Roi Dayan 2 participants 4 messages May 25, 2014 Kbuild

lore
Share & tools
lore

Roi Dayan

author
#0 lore raw

cover iser: configurable bufsize and ipv6 support

Hi,

The first patch is to make the rdma buf size configurable
which will allow to support bigger block sizes if required
or to allocate less memory if there is no need to support high block sizes. i.e. always 1K-4K IOPs.

The second patch adds support for portals which adds IPv6 support.

Thanks,
Roi

Roi Dayan (2):
iser: make membuf size configurable
iser: add portals support

 usr/iscsi/iser.c |  125 ++++++++++++++++++++++++++++++++++++++++++------------
 usr/iscsi/iser.h |    8 +++
 2 files changed, 105 insertions(+), 28 deletions(-)
author
#1 lore raw

PATCH 1/2 iser: make membuf size configurable

membuf size is the max size of one rdma buffer used. The default is still 512 as was but a user can now change it to 1024 for example to be able to use bigger block sizes.

---
 usr/iscsi/iser.c |   15 +++++++++++----
 1 files changed, 11 insertions(+), 4 deletions(-)
usr/iscsi/iser.c +11 -4
index d748184..35c6faa 100644
@@ -76,7 +76,7 @@ char *iser_portal_addr;
  */
 #define MAX_WQE 1800
 
-#define RDMA_TRANSFER_SIZE  (512 * 1024)
+#define DEFAULT_RDMA_TRANSFER_SIZE  (512 * 1024)
 
 #define MAX_POLL_WC 32
 
@@ -97,7 +97,7 @@ static size_t buf_pool_sz_mb = DEFAULT_POOL_SIZE_MB;
 static int cq_vector = -1;
 
 static int membuf_num;
-static size_t membuf_size = RDMA_TRANSFER_SIZE;
+static size_t membuf_size = DEFAULT_RDMA_TRANSFER_SIZE;
 
 static int iser_conn_get(struct iser_conn *conn);
 static int iser_conn_getn(struct iser_conn *conn, int n);
@@ -1166,8 +1166,7 @@ int iser_login_complete(struct iscsi_connection *iscsi_conn)
 
 	/* How much data to grab in an RDMA operation, read or write */
 	/* ToDo: fix iscsi login code, broken handling of MAX_XMIT_DL */
-	iscsi_conn->session_param[ISCSI_PARAM_MAX_XMIT_DLENGTH].val =
-		RDMA_TRANSFER_SIZE;
+	iscsi_conn->session_param[ISCSI_PARAM_MAX_XMIT_DLENGTH].val = membuf_size;
 out:
 	return err;
 }
@@ -3492,6 +3491,7 @@ static const char *lld_param_nop = "nop";
 static const char *lld_param_on = "on";
 static const char *lld_param_off = "off";
 static const char *lld_param_pool_sz_mb = "pool_sz_mb";
+static const char *lld_param_buf_sz_kb = "buf_sz_kb";
 static const char *lld_param_cq_vector = "cq_vector";
 
 static int iser_param_parser(char *p)
@@ -3531,6 +3531,13 @@ static int iser_param_parser(char *p)
 			buf_pool_sz_mb = atoi(q);
 			if (buf_pool_sz_mb < 128)
 				buf_pool_sz_mb = 128;
+		} else if (!strncmp(p, lld_param_buf_sz_kb,
+				    strlen(lld_param_buf_sz_kb))) {
+			q = p + strlen(lld_param_buf_sz_kb) + 1;
+			membuf_size = atoi(q);
+			if (membuf_size < 1)
+				membuf_size = 1;
+			membuf_size = membuf_size * 1024;
 		} else if (!strncmp(p, lld_param_cq_vector,
 				    strlen(lld_param_cq_vector))) {
 			q = p + strlen(lld_param_cq_vector) + 1;
Signature · 1 line
1.7.1
author
#2 lore raw

PATCH 2/2 iser: add portals support

In reply to Roi Dayan

This adds IPv6 support for iser.

---
 usr/iscsi/iser.c |  110 ++++++++++++++++++++++++++++++++++++++++++------------
 usr/iscsi/iser.h |    8 ++++
 2 files changed, 94 insertions(+), 24 deletions(-)
usr/iscsi/iser.c +86 -24
index 35c6faa..ace8da3 100644
@@ -51,7 +51,8 @@ static struct iscsi_transport iscsi_iser;
 
 /* global, across all devices */
 static struct rdma_event_channel *rdma_evt_channel;
-static struct rdma_cm_id *cma_listen_id;
+
+LIST_HEAD(iser_portals_list);
 
 /* accepted at RDMA layer, but not yet established */
 static LIST_HEAD(temp_conn);
@@ -3338,19 +3339,31 @@ static void iser_device_release(struct iser_device *dev)
 		eprintf("ibv_dealloc_pd failed: (errno=%d %m)\n", errno);
 }
 
-/*
- * Init entire iscsi transport.  Begin listening for connections.
- */
-static int iser_ib_init(void)
+static int iser_add_portal(struct addrinfo *res, short int port)
 {
 	int err;
-	struct sockaddr_in sock_addr;
-	short int port = iser_listen_port;
+	int afonly = 1;
+	struct sockaddr_storage sock_addr;
+	struct rdma_cm_id *cma_listen_id;
+	struct iser_portal *portal = NULL;
 
-	rdma_evt_channel = rdma_create_event_channel();
-	if (!rdma_evt_channel) {
-		eprintf("Failed to initialize RDMA; load kernel modules?\n");
-		return -1;
+	portal = zalloc(sizeof(struct iser_portal));
+	if (!portal) {
+	    eprintf("zalloc failed.\n");
+	    return -1;
+	}
+	portal->af = res->ai_family;
+	portal->port = port;
+
+	memset(&sock_addr, 0, sizeof(sock_addr));
+	if (res->ai_family == AF_INET6) {
+		((struct sockaddr_in6 *) &sock_addr)->sin6_family = AF_INET6;
+		((struct sockaddr_in6 *) &sock_addr)->sin6_port = htons(port);
+		((struct sockaddr_in6 *) &sock_addr)->sin6_addr = in6addr_any;
+	} else {
+		((struct sockaddr_in *) &sock_addr)->sin_family = AF_INET;
+		((struct sockaddr_in *) &sock_addr)->sin_port = htons(port);
+		((struct sockaddr_in *) &sock_addr)->sin_addr.s_addr = INADDR_ANY;
 	}
 
 	err = rdma_create_id(rdma_evt_channel, &cma_listen_id, NULL,
@@ -3359,11 +3372,11 @@ static int iser_ib_init(void)
 		eprintf("rdma_create_id failed, %m\n");
 		return -1;
 	}
+	portal->cma_listen_id = cma_listen_id;
+
+	rdma_set_option(cma_listen_id, RDMA_OPTION_ID,
+			RDMA_OPTION_ID_AFONLY, &afonly, sizeof(afonly));
 
-	memset(&sock_addr, 0, sizeof(sock_addr));
-	sock_addr.sin_family = AF_INET;
-	sock_addr.sin_port = htons(port);
-	sock_addr.sin_addr.s_addr = INADDR_ANY;
 	err =
 	    rdma_bind_addr(cma_listen_id, (struct sockaddr *) &sock_addr);
 	if (err) {
@@ -3384,8 +3397,47 @@ static int iser_ib_init(void)
 		return -1;
 	}
 
+        list_add(&portal->iser_portal_siblings, &iser_portals_list);
+	return err;
+}
+
+/*
+ * Init entire iscsi transport.  Begin listening for connections.
+ */
+static int iser_ib_init(void)
+{
+	int err;
+	short int port = iser_listen_port;
+	struct addrinfo hints, *res, *res0;
+	char servname[64];
+
+	rdma_evt_channel = rdma_create_event_channel();
+	if (!rdma_evt_channel) {
+		eprintf("Failed to initialize RDMA; load kernel modules?\n");
+		return -1;
+	}
+
+	memset(servname, 0, sizeof(servname));
+	snprintf(servname, sizeof(servname), "%d", port);
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_PASSIVE;
+
+	err = getaddrinfo(NULL, servname, &hints, &res0);
+	if (err) {
+		eprintf("unable to get address info, %m\n");
+		return -errno;
+	}
+
+	for (res = res0; res; res = res->ai_next) {
+		err = iser_add_portal(res, port);
+		if (err)
+		    return err;
+	}
+
 	dprintf("listening for iser connections on port %d\n", port);
-	err = tgt_event_add(cma_listen_id->channel->fd, EPOLLIN,
+	err = tgt_event_add(rdma_evt_channel->fd, EPOLLIN,
 			    iser_handle_rdmacm, NULL);
 	if (err)
 		return err;
@@ -3393,9 +3445,23 @@ static int iser_ib_init(void)
 	return err;
 }
 
-static void iser_ib_release(void)
+void iser_delete_portals(void)
 {
 	int err;
+	struct iser_portal *portal, *ptmp;
+
+	list_for_each_entry_safe(portal, ptmp, &iser_portals_list,
+				 iser_portal_siblings) {
+		err = rdma_destroy_id(portal->cma_listen_id);
+		if (err)
+			eprintf("rdma_destroy_id failed: (errno=%d %m)\n", errno);
+		list_del(&portal->iser_portal_siblings);
+		free(portal);
+	}
+}
+
+static void iser_ib_release(void)
+{
 	struct iser_device *dev, *tdev;
 
 	assert(list_empty(&iser_conn_list));
@@ -3405,13 +3471,9 @@ static void iser_ib_release(void)
 		free(dev);
 	}
 
-	if (cma_listen_id) {
-		tgt_event_del(cma_listen_id->channel->fd);
-
-		err = rdma_destroy_id(cma_listen_id);
-		if (err)
-			eprintf("rdma_destroy_id failed: (errno=%d %m)\n", errno);
-
+	if (!list_empty(&iser_portals_list)) {
+		tgt_event_del(rdma_evt_channel->fd);
+		iser_delete_portals();
 		rdma_destroy_event_channel(rdma_evt_channel);
 	}
 }
usr/iscsi/iser.h +8 -0
index 48d0a8c..5f1fb69 100644
@@ -30,6 +30,14 @@
 
 extern short control_port;
 
+struct iser_portal {
+	struct list_head iser_portal_siblings;
+	int port;
+	int af;
+	struct rdma_cm_id *cma_listen_id;
+};
+
+
 /*
  * The IB-extended version from the kernel.  Stags and VAs are in
  * big-endian format.
Signature · 1 line
1.7.1
#3 lore raw
In reply to Roi Dayan
Quoted 13 lines · On Sun, 25 May 2014 14:10:24 +0300 Roi Dayan <[email protected]> wrote:

Hi,

The first patch is to make the rdma buf size configurable
which will allow to support bigger block sizes if required
or to allocate less memory if there is no need to support high block sizes. i.e. always 1K-4K IOPs.

The second patch adds support for portals which adds IPv6 support.

Thanks,
Roi

Quoted 7 lines

Roi Dayan (2):
iser: make membuf size configurable
iser: add portals support

 usr/iscsi/iser.c |  125 ++++++++++++++++++++++++++++++++++++++++++------------
 usr/iscsi/iser.h |    8 +++
 2 files changed, 105 insertions(+), 28 deletions(-)

Applied, thanks a lot!