Skip to contentSkip to conversation

Beginning of multi-rail support for drivers/staging/lustre

PATCH In review NeilBrown 4 participants 98 messages Sep 7, 2018 – Sep 29, 2018 Staging Page 2 of 2

lore
Share & tools
lore
author
#81 lore raw

PATCH 28/34 lnet: add checks to ensure network interface names are unique.

In reply to NeilBrown

This is part of
8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
LU-7734 lnet: Multi-Rail local NI split

---
 .../staging/lustre/include/linux/lnet/lib-lnet.h   |    1 +
 drivers/staging/lustre/lnet/lnet/api-ni.c          |    8 ++++++
 drivers/staging/lustre/lnet/lnet/config.c          |   25 ++++++++++++++++++++
 3 files changed, 34 insertions(+)
drivers/staging/lustre/include/linux/lnet/lib-lnet.h +1 -0
index 905213fc16c7..ef551b571935 100644
@@ -632,6 +632,7 @@ int lnet_parse_routes(char *route_str, int *im_a_router);
 int lnet_parse_networks(struct list_head *nilist, char *networks);
 bool lnet_net_unique(__u32 net_id, struct list_head *nilist,
 		     struct lnet_net **net);
+bool lnet_ni_unique_net(struct list_head *nilist, char *iface);
 
 int lnet_nid2peer_locked(struct lnet_peer **lpp, lnet_nid_t nid, int cpt);
 struct lnet_peer *lnet_find_peer_locked(struct lnet_peer_table *ptable,
drivers/staging/lustre/lnet/lnet/api-ni.c +8 -0
index 042ab0d9e318..3f6f5ead8a03 100644
@@ -1433,6 +1433,14 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
 				ni_netlist);
 		list_del_init(&ni->ni_netlist);
 
+		/* make sure that the the NI we're about to start
+		 * up is actually unique. if it's not fail. */
+		if (!lnet_ni_unique_net(&net_l->net_ni_list,
+					ni->ni_interfaces[0])) {
+			rc = -EINVAL;
+			goto failed1;
+		}
+
 		/* adjust the pointer the parent network, just in case it
 		 * the net is a duplicate */
 		ni->ni_net = net_l;
drivers/staging/lustre/lnet/lnet/config.c +25 -0
index fcae50676422..11d6dbc80507 100644
@@ -95,6 +95,25 @@ lnet_net_unique(__u32 net_id, struct list_head *netlist,
 	return true;
 }
 
+/* check that the NI is unique within the list of NIs already added to
+ * a network */
+bool
+lnet_ni_unique_net(struct list_head *nilist, char *iface)
+{
+	struct list_head *tmp;
+	struct lnet_ni *ni;
+
+	list_for_each(tmp, nilist) {
+		ni = list_entry(tmp, struct lnet_ni, ni_netlist);
+
+		if (ni->ni_interfaces[0] != NULL &&
+		    strncmp(ni->ni_interfaces[0], iface, strlen(iface)) == 0)
+			return false;
+	}
+
+	return true;
+}
+
 static bool
 in_array(__u32 *array, __u32 size, __u32 value)
 {
@@ -352,6 +371,12 @@ lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface)
 	int			rc;
 	int			i;
 
+	if (iface != NULL)
+		/* make sure that this NI is unique in the net it's
+		 * being added to */
+		if (!lnet_ni_unique_net(&net->net_ni_added, iface))
+			return NULL;
+
 	ni = kzalloc(sizeof(*ni), GFP_KERNEL);
 	if (ni == NULL) {
 		CERROR("Out of memory creating network interface %s%s\n",
#82 lore raw

PATCH 28/34 lnet: add checks to ensure network interface names are unique.

Doug

?On 9/6/18, 5:55 PM, "NeilBrown" <[email protected]> wrote:

    This is part of
        8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
           LU-7734 lnet: Multi-Rail local NI split
    
    Signed-off-by: NeilBrown <[email protected]>
    ---
     .../staging/lustre/include/linux/lnet/lib-lnet.h   |    1 +
     drivers/staging/lustre/lnet/lnet/api-ni.c          |    8 ++++++
     drivers/staging/lustre/lnet/lnet/config.c          |   25 ++++++++++++++++++++
     3 files changed, 34 insertions(+)
    
    diff --git a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
    index 905213fc16c7..ef551b571935 100644
    --- a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
    +++ b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
    @@ -632,6 +632,7 @@ int lnet_parse_routes(char *route_str, int *im_a_router);
     int lnet_parse_networks(struct list_head *nilist, char *networks);
     bool lnet_net_unique(__u32 net_id, struct list_head *nilist,
     		     struct lnet_net **net);
    +bool lnet_ni_unique_net(struct list_head *nilist, char *iface);
     
     int lnet_nid2peer_locked(struct lnet_peer **lpp, lnet_nid_t nid, int cpt);
     struct lnet_peer *lnet_find_peer_locked(struct lnet_peer_table *ptable,
    diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
    index 042ab0d9e318..3f6f5ead8a03 100644
    --- a/drivers/staging/lustre/lnet/lnet/api-ni.c
    +++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
    @@ -1433,6 +1433,14 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
     				ni_netlist);
     		list_del_init(&ni->ni_netlist);
     
    +		/* make sure that the the NI we're about to start
    +		 * up is actually unique. if it's not fail. */
    +		if (!lnet_ni_unique_net(&net_l->net_ni_list,
    +					ni->ni_interfaces[0])) {
    +			rc = -EINVAL;
    +			goto failed1;
    +		}
    +
     		/* adjust the pointer the parent network, just in case it
     		 * the net is a duplicate */
     		ni->ni_net = net_l;
    diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c
    index fcae50676422..11d6dbc80507 100644
    --- a/drivers/staging/lustre/lnet/lnet/config.c
    +++ b/drivers/staging/lustre/lnet/lnet/config.c
    @@ -95,6 +95,25 @@ lnet_net_unique(__u32 net_id, struct list_head *netlist,
     	return true;
     }
     
    +/* check that the NI is unique within the list of NIs already added to
    + * a network */
    +bool
    +lnet_ni_unique_net(struct list_head *nilist, char *iface)
    +{
    +	struct list_head *tmp;
    +	struct lnet_ni *ni;
    +
    +	list_for_each(tmp, nilist) {
    +		ni = list_entry(tmp, struct lnet_ni, ni_netlist);
    +
    +		if (ni->ni_interfaces[0] != NULL &&
    +		    strncmp(ni->ni_interfaces[0], iface, strlen(iface)) == 0)
    +			return false;
    +	}
    +
    +	return true;
    +}
    +
     static bool
     in_array(__u32 *array, __u32 size, __u32 value)
     {
    @@ -352,6 +371,12 @@ lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface)
     	int			rc;
     	int			i;
     
    +	if (iface != NULL)
    +		/* make sure that this NI is unique in the net it's
    +		 * being added to */
    +		if (!lnet_ni_unique_net(&net->net_ni_added, iface))
    +			return NULL;
    +
     	ni = kzalloc(sizeof(*ni), GFP_KERNEL);
     	if (ni == NULL) {
     		CERROR("Out of memory creating network interface %s%s\n",
author
#83 lore raw

PATCH 29/34 lnet: track tunables in lnet_startup_lndnet()

In reply to NeilBrown

Not really sure what this is yet.

This is part of
8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
LU-7734 lnet: Multi-Rail local NI split

---
 drivers/staging/lustre/lnet/lnet/api-ni.c |   24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
drivers/staging/lustre/lnet/lnet/api-ni.c +24 -0
index 3f6f5ead8a03..f4efb48c4cf3 100644
@@ -1361,6 +1361,12 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
 	int			ni_count = 0;
 	__u32			lnd_type;
 	struct lnet_lnd		*lnd;
+	int			peer_timeout =
+		net->net_tunables.lct_peer_timeout;
+	int			maxtxcredits =
+		net->net_tunables.lct_max_tx_credits;
+	int			peerrtrcredits =
+		net->net_tunables.lct_peer_rtr_credits;
 
 	INIT_LIST_HEAD(&local_ni_list);
 
@@ -1447,6 +1453,9 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
 
 		rc = lnet_startup_lndni(ni, tun);
 
+		LASSERT(ni->ni_net->net_tunables.lct_peer_timeout <= 0 ||
+			ni->ni_net->net_lnd->lnd_query != NULL);
+
 		if (rc < 0)
 			goto failed1;
 
@@ -1464,8 +1473,23 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
 	 * it around after we're done. Free it. Otherwise add that
 	 * net to the global the_lnet.ln_nets */
 	if (net_l != net && net_l != NULL) {
+		/*
+		 * TODO - note. currently the tunables can not be updated
+		 * once added
+		 */
 		lnet_net_free(net);
 	} else {
+		/*
+		 * restore tunables after it has been overwitten by the
+		 * lnd
+		 */
+		if (peer_timeout != -1)
+			net->net_tunables.lct_peer_timeout = peer_timeout;
+		if (maxtxcredits != -1)
+			net->net_tunables.lct_max_tx_credits = maxtxcredits;
+		if (peerrtrcredits != -1)
+			net->net_tunables.lct_peer_rtr_credits = peerrtrcredits;
+
 		lnet_net_lock(LNET_LOCK_EX);
 		list_add_tail(&net->net_list, &the_lnet.ln_nets);
 		lnet_net_unlock(LNET_LOCK_EX);
#84 lore raw

PATCH 29/34 lnet: track tunables in lnet_startup_lndnet()

Amir: You need to comment on this one.

It looks to me that it starts off by making a copy of three tunables to see if the user has set them specific to the NET. Then, if the LND changes them, it changes them back to what the user set. What I don't get: why would the LND change the values if they are not -1?

Doug

?On 9/6/18, 5:55 PM, "NeilBrown" <[email protected]> wrote:

    Not really sure what this is yet.
    
    This is part of
        8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
           LU-7734 lnet: Multi-Rail local NI split
    
    Signed-off-by: NeilBrown <[email protected]>
    ---
     drivers/staging/lustre/lnet/lnet/api-ni.c |   24 ++++++++++++++++++++++++
     1 file changed, 24 insertions(+)
    
    diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
    index 3f6f5ead8a03..f4efb48c4cf3 100644
    --- a/drivers/staging/lustre/lnet/lnet/api-ni.c
    +++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
    @@ -1361,6 +1361,12 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
     	int			ni_count = 0;
     	__u32			lnd_type;
     	struct lnet_lnd		*lnd;
    +	int			peer_timeout =
    +		net->net_tunables.lct_peer_timeout;
    +	int			maxtxcredits =
    +		net->net_tunables.lct_max_tx_credits;
    +	int			peerrtrcredits =
    +		net->net_tunables.lct_peer_rtr_credits;
     
     	INIT_LIST_HEAD(&local_ni_list);
     
    @@ -1447,6 +1453,9 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
     
     		rc = lnet_startup_lndni(ni, tun);
     
    +		LASSERT(ni->ni_net->net_tunables.lct_peer_timeout <= 0 ||
    +			ni->ni_net->net_lnd->lnd_query != NULL);
    +
     		if (rc < 0)
     			goto failed1;
     
    @@ -1464,8 +1473,23 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
     	 * it around after we're done. Free it. Otherwise add that
     	 * net to the global the_lnet.ln_nets */
     	if (net_l != net && net_l != NULL) {
    +		/*
    +		 * TODO - note. currently the tunables can not be updated
    +		 * once added
    +		 */
     		lnet_net_free(net);
     	} else {
    +		/*
    +		 * restore tunables after it has been overwitten by the
    +		 * lnd
    +		 */
    +		if (peer_timeout != -1)
    +			net->net_tunables.lct_peer_timeout = peer_timeout;
    +		if (maxtxcredits != -1)
    +			net->net_tunables.lct_max_tx_credits = maxtxcredits;
    +		if (peerrtrcredits != -1)
    +			net->net_tunables.lct_peer_rtr_credits = peerrtrcredits;
    +
     		lnet_net_lock(LNET_LOCK_EX);
     		list_add_tail(&net->net_list, &the_lnet.ln_nets);
     		lnet_net_unlock(LNET_LOCK_EX);
author
#85 lore raw

PATCH 30/34 lnet: fix typo

In reply to NeilBrown

to -> too

This is part of
8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
LU-7734 lnet: Multi-Rail local NI split

---
 drivers/staging/lustre/lnet/lnet/api-ni.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/staging/lustre/lnet/lnet/api-ni.c +1 -1
index f4efb48c4cf3..cf0ffb8ac84b 100644
@@ -1868,7 +1868,7 @@ lnet_fill_ni_info(struct lnet_ni *ni, struct lnet_ioctl_config_data *config)
 	if (config->cfg_hdr.ioc_len > min_size)
 		tunable_size = config->cfg_hdr.ioc_len - min_size;
 
-	/* Don't copy to much data to user space */
+	/* Don't copy too much data to user space */
 	min_size = min(tunable_size, sizeof(ni->ni_lnd_tunables));
 	lnd_cfg = (struct lnet_ioctl_config_lnd_tunables *)net_config->cfg_bulk;
 
#86 lore raw

PATCH 30/34 lnet: fix typo

Doug

?On 9/6/18, 5:55 PM, "NeilBrown" <[email protected]> wrote:

    to -> too
    
    This is part of
        8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
           LU-7734 lnet: Multi-Rail local NI split
    
    Signed-off-by: NeilBrown <[email protected]>
    ---
     drivers/staging/lustre/lnet/lnet/api-ni.c |    2 +-
     1 file changed, 1 insertion(+), 1 deletion(-)
    
    diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
    index f4efb48c4cf3..cf0ffb8ac84b 100644
    --- a/drivers/staging/lustre/lnet/lnet/api-ni.c
    +++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
    @@ -1868,7 +1868,7 @@ lnet_fill_ni_info(struct lnet_ni *ni, struct lnet_ioctl_config_data *config)
     	if (config->cfg_hdr.ioc_len > min_size)
     		tunable_size = config->cfg_hdr.ioc_len - min_size;
     
    -	/* Don't copy to much data to user space */
    +	/* Don't copy too much data to user space */
     	min_size = min(tunable_size, sizeof(ni->ni_lnd_tunables));
     	lnd_cfg = (struct lnet_ioctl_config_lnd_tunables *)net_config->cfg_bulk;
author
#87 lore raw

PATCH 31/34 lnet: lnet_dyn_add_ni: fix ping_info count

In reply to NeilBrown

Use the correct count of interfaces when calling
lnet_ping_info_setup()
in lnet_dyn_add_ni()

---
 drivers/staging/lustre/lnet/lnet/api-ni.c |   27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)
drivers/staging/lustre/lnet/lnet/api-ni.c +26 -1
index cf0ffb8ac84b..2ce0a7212dc2 100644
@@ -871,6 +871,18 @@ lnet_ping_info_create(int num_ni)
 	return ping_info;
 }
 
+static inline int
+lnet_get_net_ni_count_locked(struct lnet_net *net)
+{
+	struct lnet_ni	*ni;
+	int		count = 0;
+
+	list_for_each_entry(ni, &net->net_ni_list, ni_netlist)
+		count++;
+
+	return count;
+}
+
 static inline int
 lnet_get_ni_count(void)
 {
@@ -1977,6 +1989,7 @@ lnet_dyn_add_ni(lnet_pid_t requested_pid, struct lnet_ioctl_config_data *conf)
 	struct list_head net_head;
 	struct lnet_remotenet *rnet;
 	int rc;
+	int			net_ni_count;
 	int			num_acceptor_nets;
 	__u32			net_type;
 	struct lnet_ioctl_config_lnd_tunables *lnd_tunables = NULL;
@@ -2014,7 +2027,19 @@ lnet_dyn_add_ni(lnet_pid_t requested_pid, struct lnet_ioctl_config_data *conf)
 		goto failed0;
 	}
 
-	rc = lnet_ping_info_setup(&pinfo, &md_handle, 1 + lnet_get_ni_count(),
+	/*
+	 * make sure you calculate the correct number of slots in the ping
+	 * info. Since the ping info is a flattened list of all the NIs,
+	 * we should allocate enough slots to accomodate the number of NIs
+	 * which will be added.
+	 *
+	 * We can use lnet_get_net_ni_count_locked() since the net is not
+	 * on a public list yet, so locking is not a problem
+	 */
+	net_ni_count = lnet_get_net_ni_count_locked(net);
+
+	rc = lnet_ping_info_setup(&pinfo, &md_handle,
+				  net_ni_count + lnet_get_ni_count(),
 				  false);
 	if (rc)
 		goto failed0;
#88 lore raw

PATCH 31/34 lnet: lnet_dyn_add_ni: fix ping_info count

Doug

?On 9/6/18, 5:55 PM, "NeilBrown" <[email protected]> wrote:

    Use the correct count of interfaces when calling
       lnet_ping_info_setup()
    in lnet_dyn_add_ni()
    
    Signed-off-by: NeilBrown <[email protected]>
    ---
     drivers/staging/lustre/lnet/lnet/api-ni.c |   27 ++++++++++++++++++++++++++-
     1 file changed, 26 insertions(+), 1 deletion(-)
    
    diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
    index cf0ffb8ac84b..2ce0a7212dc2 100644
    --- a/drivers/staging/lustre/lnet/lnet/api-ni.c
    +++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
    @@ -871,6 +871,18 @@ lnet_ping_info_create(int num_ni)
     	return ping_info;
     }
     
    +static inline int
    +lnet_get_net_ni_count_locked(struct lnet_net *net)
    +{
    +	struct lnet_ni	*ni;
    +	int		count = 0;
    +
    +	list_for_each_entry(ni, &net->net_ni_list, ni_netlist)
    +		count++;
    +
    +	return count;
    +}
    +
     static inline int
     lnet_get_ni_count(void)
     {
    @@ -1977,6 +1989,7 @@ lnet_dyn_add_ni(lnet_pid_t requested_pid, struct lnet_ioctl_config_data *conf)
     	struct list_head net_head;
     	struct lnet_remotenet *rnet;
     	int rc;
    +	int			net_ni_count;
     	int			num_acceptor_nets;
     	__u32			net_type;
     	struct lnet_ioctl_config_lnd_tunables *lnd_tunables = NULL;
    @@ -2014,7 +2027,19 @@ lnet_dyn_add_ni(lnet_pid_t requested_pid, struct lnet_ioctl_config_data *conf)
     		goto failed0;
     	}
     
    -	rc = lnet_ping_info_setup(&pinfo, &md_handle, 1 + lnet_get_ni_count(),
    +	/*
    +	 * make sure you calculate the correct number of slots in the ping
    +	 * info. Since the ping info is a flattened list of all the NIs,
    +	 * we should allocate enough slots to accomodate the number of NIs
    +	 * which will be added.
    +	 *
    +	 * We can use lnet_get_net_ni_count_locked() since the net is not
    +	 * on a public list yet, so locking is not a problem
    +	 */
    +	net_ni_count = lnet_get_net_ni_count_locked(net);
    +
    +	rc = lnet_ping_info_setup(&pinfo, &md_handle,
    +				  net_ni_count + lnet_get_ni_count(),
     				  false);
     	if (rc)
     		goto failed0;
author
#89 lore raw

PATCH 32/34 lnet: lnet_dyn_del_ni: fix ping_info count

In reply to NeilBrown

- use correct interface count for lnet_ping_info_setup().
- also rename 'net' to 'net_id' so the name 'net' is free to identify the lnet_net.

This is part of
8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
LU-7734 lnet: Multi-Rail local NI split

---
 drivers/staging/lustre/lnet/lnet/api-ni.c |   35 +++++++++++++++++------------
 1 file changed, 20 insertions(+), 15 deletions(-)
drivers/staging/lustre/lnet/lnet/api-ni.c +20 -15
index 2ce0a7212dc2..ff5149da2d79 100644
@@ -2109,40 +2109,45 @@ lnet_dyn_add_ni(lnet_pid_t requested_pid, struct lnet_ioctl_config_data *conf)
 }
 
 int
-lnet_dyn_del_ni(__u32 net)
+lnet_dyn_del_ni(__u32 net_id)
 {
-	struct lnet_ni *ni;
+	struct lnet_net *net;
 	struct lnet_ping_info *pinfo;
 	struct lnet_handle_md md_handle;
 	int rc;
+	int		  net_ni_count;
 
 	/* don't allow userspace to shutdown the LOLND */
-	if (LNET_NETTYP(net) == LOLND)
+	if (LNET_NETTYP(net_id) == LOLND)
 		return -EINVAL;
 
 	mutex_lock(&the_lnet.ln_api_mutex);
+
+	lnet_net_lock(0);
+
+	net = lnet_get_net_locked(net_id);
+	if (net == NULL) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	net_ni_count = lnet_get_net_ni_count_locked(net);
+
+	lnet_net_unlock(0);
+
 	/* create and link a new ping info, before removing the old one */
 	rc = lnet_ping_info_setup(&pinfo, &md_handle,
-				  lnet_get_ni_count() - 1, false);
+				  lnet_get_ni_count() - net_ni_count, false);
 	if (rc)
 		goto out;
 
-	ni = lnet_net2ni(net);
-	if (!ni) {
-		rc = -EINVAL;
-		goto failed;
-	}
-
-	lnet_shutdown_lndni(ni);
+	lnet_shutdown_lndnet(net);
 
 	if (!lnet_count_acceptor_nets())
 		lnet_acceptor_stop();
 
 	lnet_ping_target_update(pinfo, md_handle);
-	goto out;
-failed:
-	lnet_ping_md_unlink(pinfo, &md_handle);
-	lnet_ping_info_free(pinfo);
+
 out:
 	mutex_unlock(&the_lnet.ln_api_mutex);
 
#90 lore raw

PATCH 32/34 lnet: lnet_dyn_del_ni: fix ping_info count

Doug

?On 9/6/18, 5:55 PM, "NeilBrown" <[email protected]> wrote:

    - use correct interface count for lnet_ping_info_setup().
    - also rename 'net' to 'net_id' so the name 'net' is free
      to identify the lnet_net.
    
    This is part of
        8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
           LU-7734 lnet: Multi-Rail local NI split
    
    Signed-off-by: NeilBrown <[email protected]>
    ---
     drivers/staging/lustre/lnet/lnet/api-ni.c |   35 +++++++++++++++++------------
     1 file changed, 20 insertions(+), 15 deletions(-)
    
    diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
    index 2ce0a7212dc2..ff5149da2d79 100644
    --- a/drivers/staging/lustre/lnet/lnet/api-ni.c
    +++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
    @@ -2109,40 +2109,45 @@ lnet_dyn_add_ni(lnet_pid_t requested_pid, struct lnet_ioctl_config_data *conf)
     }
     
     int
    -lnet_dyn_del_ni(__u32 net)
    +lnet_dyn_del_ni(__u32 net_id)
     {
    -	struct lnet_ni *ni;
    +	struct lnet_net *net;
     	struct lnet_ping_info *pinfo;
     	struct lnet_handle_md md_handle;
     	int rc;
    +	int		  net_ni_count;
     
     	/* don't allow userspace to shutdown the LOLND */
    -	if (LNET_NETTYP(net) == LOLND)
    +	if (LNET_NETTYP(net_id) == LOLND)
     		return -EINVAL;
     
     	mutex_lock(&the_lnet.ln_api_mutex);
    +
    +	lnet_net_lock(0);
    +
    +	net = lnet_get_net_locked(net_id);
    +	if (net == NULL) {
    +		rc = -EINVAL;
    +		goto out;
    +	}
    +
    +	net_ni_count = lnet_get_net_ni_count_locked(net);
    +
    +	lnet_net_unlock(0);
    +
     	/* create and link a new ping info, before removing the old one */
     	rc = lnet_ping_info_setup(&pinfo, &md_handle,
    -				  lnet_get_ni_count() - 1, false);
    +				  lnet_get_ni_count() - net_ni_count, false);
     	if (rc)
     		goto out;
     
    -	ni = lnet_net2ni(net);
    -	if (!ni) {
    -		rc = -EINVAL;
    -		goto failed;
    -	}
    -
    -	lnet_shutdown_lndni(ni);
    +	lnet_shutdown_lndnet(net);
     
     	if (!lnet_count_acceptor_nets())
     		lnet_acceptor_stop();
     
     	lnet_ping_target_update(pinfo, md_handle);
    -	goto out;
    -failed:
    -	lnet_ping_md_unlink(pinfo, &md_handle);
    -	lnet_ping_info_free(pinfo);
    +
     out:
     	mutex_unlock(&the_lnet.ln_api_mutex);
author
#91 lore raw

PATCH 33/34 Completely re-write lnet_parse_networks().

In reply to NeilBrown

From: Amir Shehata <[email protected]>

Was:

LU-7734 lnet: Multi-Rail local NI split

This patch allows the configuration of multiple NIs under one Net. It is now possible to have multiple NIDs on the same network:
Ex: <ip1>@tcp, <ip2>@tcp.
This can be configured using the following syntax:
Ex: tcp(eth0, eth1)

The data structures for the example above can be visualized as follows

               NET(tcp)
                |
        -----------------
        |               |
      NI(eth0)        NI(eth1)

For more details refer to the Mult-Rail Requirements and HLD documents

Change-Id Id7c73b9b811a3082b61e53b9e9f95743188cbd51

Reviewed-on: http://review.whamcloud.com/18274

---
 drivers/staging/lustre/lnet/lnet/config.c |  341 ++++++++++++++++++-----------
 1 file changed, 217 insertions(+), 124 deletions(-)
drivers/staging/lustre/lnet/lnet/config.c +217 -124
index 11d6dbc80507..0571fa6a7249 100644
@@ -48,8 +48,11 @@ static int lnet_tbnob;			/* track text buf allocation */
 #define LNET_MAX_TEXTBUF_NOB     (64 << 10)	/* bound allocation */
 #define LNET_SINGLE_TEXTBUF_NOB  (4 << 10)
 
+#define SPACESTR " \t\v\r\n"
+#define DELIMITERS ":()[]"
+
 static void
-lnet_syntax(char *name, char *str, int offset, int width)
+lnet_syntax(const char *name, const char *str, int offset, int width)
 {
 	static char dots[LNET_SINGLE_TEXTBUF_NOB];
 	static char dashes[LNET_SINGLE_TEXTBUF_NOB];
@@ -363,6 +366,42 @@ lnet_net_alloc(__u32 net_id, struct list_head *net_list)
 	return net;
 }
 
+static int
+lnet_ni_add_interface(struct lnet_ni *ni, char *iface)
+{
+	int niface = 0;
+
+	if (ni == NULL)
+		return -ENOMEM;
+
+	/* Allocate a separate piece of memory and copy
+	 * into it the string, so we don't have
+	 * a depencency on the tokens string.  This way we
+	 * can free the tokens@the end of the function.
+	 * The newly allocated ni_interfaces[] can be
+	 * freed when freeing the NI */
+	while (niface < LNET_MAX_INTERFACES &&
+	       ni->ni_interfaces[niface] != NULL)
+		niface++;
+
+	if (niface >= LNET_MAX_INTERFACES) {
+		LCONSOLE_ERROR_MSG(0x115, "Too many interfaces "
+				   "for net %s\n",
+				   libcfs_net2str(LNET_NIDNET(ni->ni_nid)));
+		return -EINVAL;
+	}
+
+	ni->ni_interfaces[niface] = kstrdup(iface, GFP_KERNEL);
+
+	if (ni->ni_interfaces[niface] == NULL) {
+		CERROR("Can't allocate net interface name\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/* allocate and add to the provided network */
 struct lnet_ni *
 lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface)
 {
@@ -439,24 +478,33 @@ lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface)
 		goto failed;
 	list_add_tail(&ni->ni_netlist, &net->net_ni_added);
 
+	/* if an interface name is provided then make sure to add in that
+	 * interface name in NI */
+	if (iface != NULL)
+		if (lnet_ni_add_interface(ni, iface) != 0)
+			goto failed;
+
 	return ni;
 failed:
 	lnet_ni_free(ni);
 	return NULL;
 }
 
+/*
+ * Parse the networks string and create the matching set of NIs on the
+ * nilist.
+ */
 int
 lnet_parse_networks(struct list_head *netlist, char *networks)
 {
-	struct cfs_expr_list *el = NULL;
+	struct cfs_expr_list *net_el = NULL;
+	struct cfs_expr_list *ni_el = NULL;
 	char *tokens;
 	char *str;
-	char *tmp;
 	struct lnet_net *net;
 	struct lnet_ni *ni = NULL;
 	__u32 net_id;
 	int nnets = 0;
-	struct list_head *temp_node;
 
 	if (!networks) {
 		CERROR("networks string is undefined\n");
@@ -476,84 +524,108 @@ lnet_parse_networks(struct list_head *netlist, char *networks)
 		return -ENOMEM;
 	}
 
-	tmp = tokens;
 	str = tokens;
 
-	while (str && *str) {
-		char *comma = strchr(str, ',');
-		char *bracket = strchr(str, '(');
-		char *square = strchr(str, '[');
-		char *iface;
-		int niface;
+	/*
+	 * Main parser loop.
+	 *
+	 * NB we don't check interface conflicts here; it's the LNDs
+	 * responsibility (if it cares at all)
+	 */
+	do {
+		char *nistr;
+		char *elstr;
+		char *name;
 		int rc;
 
 		/*
-		 * NB we don't check interface conflicts here; it's the LNDs
-		 * responsibility (if it cares at all)
+		 * Parse a network string into its components.
+		 *
+		 * <name>{"("...")"}{"["<el>"]"}
 		 */
-		if (square && (!comma || square < comma)) {
-			/*
-			 * i.e: o2ib0(ib0)[1,2], number between square
-			 * brackets are CPTs this NI needs to be bond
-			 */
-			if (bracket && bracket > square) {
-				tmp = square;
+
+		/* Network name (mandatory)
+		 */
+		while (isspace(*str))
+			*str++ = '\0';
+		if (!*str)
+			break;
+		name = str;
+		str += strcspn(str, SPACESTR ":()[],");
+		while (isspace(*str))
+			*str++ = '\0';
+
+		/* Interface list (optional) */
+		if (*str == '(') {
+			*str++ = '\0';
+			nistr = str;
+			str += strcspn(str, ")");
+			if (*str != ')') {
+				str = nistr;
 				goto failed_syntax;
 			}
+			do {
+				*str++ = '\0';
+			} while (isspace(*str));
+		} else {
+			nistr = NULL;
+		}
 
-			tmp = strchr(square, ']');
-			if (!tmp) {
-				tmp = square;
+		/* CPT expression (optional) */
+		if (*str == '[') {
+			elstr = str;
+			str += strcspn(str, "]");
+			if (*str != ']') {
+				str = elstr;
 				goto failed_syntax;
 			}
-
-			rc = cfs_expr_list_parse(square, tmp - square + 1,
-						 0, LNET_CPT_NUMBER - 1, &el);
+			rc = cfs_expr_list_parse(elstr, str - elstr + 1,
+						0, LNET_CPT_NUMBER - 1,
+						&net_el);
 			if (rc) {
-				tmp = square;
+				str = elstr;
 				goto failed_syntax;
 			}
-
-			while (square <= tmp)
-				*square++ = ' ';
+			*elstr = '\0';
+			do {
+				*str++ = '\0';
+			} while (isspace(*str));
 		}
 
-		if (!bracket || (comma && comma < bracket)) {
-			/* no interface list specified */
+		/* Bad delimiters */
+		if (*str && (strchr(DELIMITERS, *str) != NULL))
+			goto failed_syntax;
 
-			if (comma)
-				*comma++ = 0;
-			net_id = libcfs_str2net(strim(str));
+		/* go to the next net if it exits */
+		str += strcspn(str, ",");
+		if (*str == ',')
+			*str++ = '\0';
 
-			if (net_id == LNET_NIDNET(LNET_NID_ANY)) {
-				LCONSOLE_ERROR_MSG(0x113,
-						   "Unrecognised network type\n");
-				tmp = str;
-				goto failed_syntax;
-			}
-
-			if (LNET_NETTYP(net_id) != LOLND) { /* LO is implicit */
-				net = lnet_net_alloc(net_id, netlist);
-				if (!net ||
-				    !lnet_ni_alloc(net, el, NULL))
-					goto failed;
-			}
+		/*
+		 * At this point the name is properly terminated.
+		 */
+		net_id = libcfs_str2net(name);
+		if (net_id == LNET_NIDNET(LNET_NID_ANY)) {
+			LCONSOLE_ERROR_MSG(0x113,
+					"Unrecognised network type\n");
+			str = name;
+			goto failed_syntax;
+		}
 
-			if (el) {
-				cfs_expr_list_free(el);
-				el = NULL;
+		if (LNET_NETTYP(net_id) == LOLND) {
+			/* Loopback is implicit, and there can be only one. */
+			if (net_el) {
+				cfs_expr_list_free(net_el);
+				net_el = NULL;
 			}
-
-			str = comma;
+			/* Should we error out instead? */
 			continue;
 		}
 
-		*bracket = 0;
-		net_id = libcfs_str2net(strim(str));
-		if (net_id == LNET_NIDNET(LNET_NID_ANY)) {
-			tmp = str;
-			goto failed_syntax;
-		}
+		/*
+		 * All network paramaters are now known.
+		 */
+		nnets++;
 
 		/* always allocate a net, since we will eventually add an
 		 * interface to it, or we will fail, in which case we'll
@@ -562,88 +634,107 @@ lnet_parse_networks(struct list_head *netlist, char *networks)
 		if (IS_ERR_OR_NULL(net))
 			goto failed;
 
-		ni = lnet_ni_alloc(net, el, NULL);
-		if (IS_ERR_OR_NULL(ni))
-			goto failed;
-
-		if (el) {
-			cfs_expr_list_free(el);
-			el = NULL;
-		}
-
-		niface = 0;
-		iface = bracket + 1;
+		if (!nistr) {
+			/*
+			 * No interface list was specified, allocate a
+			 * ni using the defaults.
+			 */
+			ni = lnet_ni_alloc(net, net_el, NULL);
+			if (IS_ERR_OR_NULL(ni))
+				goto failed;
 
-		bracket = strchr(iface, ')');
-		if (!bracket) {
-			tmp = iface;
-			goto failed_syntax;
+			if (net_el) {
+				cfs_expr_list_free(net_el);
+				net_el = NULL;
+			}
+			continue;
 		}
 
-		*bracket = 0;
 		do {
-			comma = strchr(iface, ',');
-			if (comma)
-				*comma++ = 0;
-
-			iface = strim(iface);
-			if (!*iface) {
-				tmp = iface;
-				goto failed_syntax;
+			elstr = NULL;
+
+			/* Interface name (mandatory) */
+			while (isspace(*nistr))
+				*nistr++ = '\0';
+			name = nistr;
+			nistr += strcspn(nistr, SPACESTR "[],");
+			while (isspace(*nistr))
+				*nistr++ = '\0';
+
+			/* CPT expression (optional) */
+			if (*nistr == '[') {
+				elstr = nistr;
+				nistr += strcspn(nistr, "]");
+				if (*nistr != ']') {
+					str = elstr;
+					goto failed_syntax;
+				}
+				rc = cfs_expr_list_parse(elstr,
+							nistr - elstr + 1,
+							0, LNET_CPT_NUMBER - 1,
+							&ni_el);
+				if (rc != 0) {
+					str = elstr;
+					goto failed_syntax;
+				}
+				*elstr = '\0';
+				do {
+					*nistr++ = '\0';
+				} while (isspace(*nistr));
+			} else {
+				ni_el = net_el;
 			}
 
-			if (niface == LNET_MAX_INTERFACES) {
-				LCONSOLE_ERROR_MSG(0x115,
-						   "Too many interfaces for net %s\n",
-						   libcfs_net2str(net_id));
-				goto failed;
+			/*
+			 * End of single interface specificaton,
+			 * advance to the start of the next one, if
+			 * any.
+			 */
+			if (*nistr == ',') {
+				do {
+					*nistr++ = '\0';
+				} while (isspace(*nistr));
+				if (!*nistr) {
+					str = nistr;
+					goto failed_syntax;
+				}
+			} else if (*nistr) {
+				str = nistr;
+				goto failed_syntax;
 			}
 
 			/*
-			 * Allocate a separate piece of memory and copy
-			 * into it the string, so we don't have
-			 * a depencency on the tokens string.  This way we
-			 * can free the tokens@the end of the function.
-			 * The newly allocated ni_interfaces[] can be
-			 * freed when freeing the NI
+			 * At this point the name
+			 is properly terminated.
 			 */
-			ni->ni_interfaces[niface] = kstrdup(iface, GFP_KERNEL);
-			if (!ni->ni_interfaces[niface]) {
-				CERROR("Can't allocate net interface name\n");
-				goto failed;
-			}
-			niface++;
-			iface = comma;
-		} while (iface);
-
-		str = bracket + 1;
-		comma = strchr(bracket + 1, ',');
-		if (comma) {
-			*comma = 0;
-			str = strim(str);
-			if (*str) {
-				tmp = str;
+			if (!*name) {
+				str = name;
 				goto failed_syntax;
 			}
-			str = comma + 1;
-			continue;
-		}
 
-		str = strim(str);
-		if (*str) {
-			tmp = str;
-			goto failed_syntax;
-		}
-	}
+			ni = lnet_ni_alloc(net, ni_el, name);
+			if (IS_ERR_OR_NULL(ni))
+				goto failed;
 
-	list_for_each(temp_node, netlist)
-		nnets++;
+			if (ni_el) {
+				if (ni_el != net_el) {
+					cfs_expr_list_free(ni_el);
+					ni_el = NULL;
+				}
+			}
+		} while (*nistr);
+
+		if (net_el) {
+			cfs_expr_list_free(net_el);
+			net_el = NULL;
+		}
+	} while (*str);
 
 	kfree(tokens);
 	return nnets;
 
  failed_syntax:
-	lnet_syntax("networks", networks, (int)(tmp - tokens), strlen(tmp));
+	lnet_syntax("networks", networks, (int)(str - tokens), strlen(str));
  failed:
 	/* free the net list and all the nis on each net */
 	while (!list_empty(netlist)) {
@@ -653,8 +744,10 @@ lnet_parse_networks(struct list_head *netlist, char *networks)
 		lnet_net_free(net);
 	}
 
-	if (el)
-		cfs_expr_list_free(el);
+	if (ni_el && ni_el != net_el)
+		cfs_expr_list_free(ni_el);
+	if (net_el)
+		cfs_expr_list_free(net_el);
 
 	kfree(tokens);
 
#92 lore raw

PATCH 33/34 Completely re-write lnet_parse_networks().

Doug

?On 9/6/18, 5:55 PM, "NeilBrown" <[email protected]> wrote:

    From: Amir Shehata <[email protected]>
    
    Was:
    
    LU-7734 lnet: Multi-Rail local NI split
    
    This patch allows the configuration of multiple NIs under one Net.
    It is now possible to have multiple NIDs on the same network:
       Ex: <ip1>@tcp, <ip2>@tcp.
    This can be configured using the following syntax:
       Ex: tcp(eth0, eth1)
    
    The data structures for the example above can be visualized
    as follows
    
                   NET(tcp)
                    |
            -----------------
            |               |
          NI(eth0)        NI(eth1)
    
    For more details refer to the Mult-Rail Requirements and HLD
    documents
    
    Signed-off-by: Amir Shehata <[email protected]>
    Change-Id: Id7c73b9b811a3082b61e53b9e9f95743188cbd51
    Reviewed-on: http://review.whamcloud.com/18274
    Tested-by: Jenkins
    Reviewed-by: Doug Oucharek <[email protected]>
    Tested-by: Maloo <[email protected]>
    Reviewed-by: Olaf Weber <[email protected]>
    ---
     drivers/staging/lustre/lnet/lnet/config.c |  341 ++++++++++++++++++-----------
     1 file changed, 217 insertions(+), 124 deletions(-)
    
    diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c
    index 11d6dbc80507..0571fa6a7249 100644
    --- a/drivers/staging/lustre/lnet/lnet/config.c
    +++ b/drivers/staging/lustre/lnet/lnet/config.c
    @@ -48,8 +48,11 @@ static int lnet_tbnob;			/* track text buf allocation */
     #define LNET_MAX_TEXTBUF_NOB     (64 << 10)	/* bound allocation */
     #define LNET_SINGLE_TEXTBUF_NOB  (4 << 10)
     
    +#define SPACESTR " \t\v\r\n"
    +#define DELIMITERS ":()[]"
    +
     static void
    -lnet_syntax(char *name, char *str, int offset, int width)
    +lnet_syntax(const char *name, const char *str, int offset, int width)
     {
     	static char dots[LNET_SINGLE_TEXTBUF_NOB];
     	static char dashes[LNET_SINGLE_TEXTBUF_NOB];
    @@ -363,6 +366,42 @@ lnet_net_alloc(__u32 net_id, struct list_head *net_list)
     	return net;
     }
     
    +static int
    +lnet_ni_add_interface(struct lnet_ni *ni, char *iface)
    +{
    +	int niface = 0;
    +
    +	if (ni == NULL)
    +		return -ENOMEM;
    +
    +	/* Allocate a separate piece of memory and copy
    +	 * into it the string, so we don't have
    +	 * a depencency on the tokens string.  This way we
    +	 * can free the tokens@the end of the function.
    +	 * The newly allocated ni_interfaces[] can be
    +	 * freed when freeing the NI */
    +	while (niface < LNET_MAX_INTERFACES &&
    +	       ni->ni_interfaces[niface] != NULL)
    +		niface++;
    +
    +	if (niface >= LNET_MAX_INTERFACES) {
    +		LCONSOLE_ERROR_MSG(0x115, "Too many interfaces "
    +				   "for net %s\n",
    +				   libcfs_net2str(LNET_NIDNET(ni->ni_nid)));
    +		return -EINVAL;
    +	}
    +
    +	ni->ni_interfaces[niface] = kstrdup(iface, GFP_KERNEL);
    +
    +	if (ni->ni_interfaces[niface] == NULL) {
    +		CERROR("Can't allocate net interface name\n");
    +		return -ENOMEM;
    +	}
    +
    +	return 0;
    +}
    +
    +/* allocate and add to the provided network */
     struct lnet_ni *
     lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface)
     {
    @@ -439,24 +478,33 @@ lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface)
     		goto failed;
     	list_add_tail(&ni->ni_netlist, &net->net_ni_added);
     
    +	/* if an interface name is provided then make sure to add in that
    +	 * interface name in NI */
    +	if (iface != NULL)
    +		if (lnet_ni_add_interface(ni, iface) != 0)
    +			goto failed;
    +
     	return ni;
     failed:
     	lnet_ni_free(ni);
     	return NULL;
     }
     
    +/*
    + * Parse the networks string and create the matching set of NIs on the
    + * nilist.
    + */
     int
     lnet_parse_networks(struct list_head *netlist, char *networks)
     {
    -	struct cfs_expr_list *el = NULL;
    +	struct cfs_expr_list *net_el = NULL;
    +	struct cfs_expr_list *ni_el = NULL;
     	char *tokens;
     	char *str;
    -	char *tmp;
     	struct lnet_net *net;
     	struct lnet_ni *ni = NULL;
     	__u32 net_id;
     	int nnets = 0;
    -	struct list_head *temp_node;
     
     	if (!networks) {
     		CERROR("networks string is undefined\n");
    @@ -476,84 +524,108 @@ lnet_parse_networks(struct list_head *netlist, char *networks)
     		return -ENOMEM;
     	}
     
    -	tmp = tokens;
     	str = tokens;
     
    -	while (str && *str) {
    -		char *comma = strchr(str, ',');
    -		char *bracket = strchr(str, '(');
    -		char *square = strchr(str, '[');
    -		char *iface;
    -		int niface;
    +	/*
    +	 * Main parser loop.
    +	 *
    +	 * NB we don't check interface conflicts here; it's the LNDs
    +	 * responsibility (if it cares at all)
    +	 */
    +	do {
    +		char *nistr;
    +		char *elstr;
    +		char *name;
     		int rc;
     
     		/*
    -		 * NB we don't check interface conflicts here; it's the LNDs
    -		 * responsibility (if it cares at all)
    +		 * Parse a network string into its components.
    +		 *
    +		 * <name>{"("...")"}{"["<el>"]"}
     		 */
    -		if (square && (!comma || square < comma)) {
    -			/*
    -			 * i.e: o2ib0(ib0)[1,2], number between square
    -			 * brackets are CPTs this NI needs to be bond
    -			 */
    -			if (bracket && bracket > square) {
    -				tmp = square;
    +
    +		/* Network name (mandatory)
    +		 */
    +		while (isspace(*str))
    +			*str++ = '\0';
    +		if (!*str)
    +			break;
    +		name = str;
    +		str += strcspn(str, SPACESTR ":()[],");
    +		while (isspace(*str))
    +			*str++ = '\0';
    +
    +		/* Interface list (optional) */
    +		if (*str == '(') {
    +			*str++ = '\0';
    +			nistr = str;
    +			str += strcspn(str, ")");
    +			if (*str != ')') {
    +				str = nistr;
     				goto failed_syntax;
     			}
    +			do {
    +				*str++ = '\0';
    +			} while (isspace(*str));
    +		} else {
    +			nistr = NULL;
    +		}
     
    -			tmp = strchr(square, ']');
    -			if (!tmp) {
    -				tmp = square;
    +		/* CPT expression (optional) */
    +		if (*str == '[') {
    +			elstr = str;
    +			str += strcspn(str, "]");
    +			if (*str != ']') {
    +				str = elstr;
     				goto failed_syntax;
     			}
    -
    -			rc = cfs_expr_list_parse(square, tmp - square + 1,
    -						 0, LNET_CPT_NUMBER - 1, &el);
    +			rc = cfs_expr_list_parse(elstr, str - elstr + 1,
    +						0, LNET_CPT_NUMBER - 1,
    +						&net_el);
     			if (rc) {
    -				tmp = square;
    +				str = elstr;
     				goto failed_syntax;
     			}
    -
    -			while (square <= tmp)
    -				*square++ = ' ';
    +			*elstr = '\0';
    +			do {
    +				*str++ = '\0';
    +			} while (isspace(*str));
     		}
     
    -		if (!bracket || (comma && comma < bracket)) {
    -			/* no interface list specified */
    +		/* Bad delimiters */
    +		if (*str && (strchr(DELIMITERS, *str) != NULL))
    +			goto failed_syntax;
     
    -			if (comma)
    -				*comma++ = 0;
    -			net_id = libcfs_str2net(strim(str));
    +		/* go to the next net if it exits */
    +		str += strcspn(str, ",");
    +		if (*str == ',')
    +			*str++ = '\0';
     
    -			if (net_id == LNET_NIDNET(LNET_NID_ANY)) {
    -				LCONSOLE_ERROR_MSG(0x113,
    -						   "Unrecognised network type\n");
    -				tmp = str;
    -				goto failed_syntax;
    -			}
    -
    -			if (LNET_NETTYP(net_id) != LOLND) { /* LO is implicit */
    -				net = lnet_net_alloc(net_id, netlist);
    -				if (!net ||
    -				    !lnet_ni_alloc(net, el, NULL))
    -					goto failed;
    -			}
    +		/*
    +		 * At this point the name is properly terminated.
    +		 */
    +		net_id = libcfs_str2net(name);
    +		if (net_id == LNET_NIDNET(LNET_NID_ANY)) {
    +			LCONSOLE_ERROR_MSG(0x113,
    +					"Unrecognised network type\n");
    +			str = name;
    +			goto failed_syntax;
    +		}
     
    -			if (el) {
    -				cfs_expr_list_free(el);
    -				el = NULL;
    +		if (LNET_NETTYP(net_id) == LOLND) {
    +			/* Loopback is implicit, and there can be only one. */
    +			if (net_el) {
    +				cfs_expr_list_free(net_el);
    +				net_el = NULL;
     			}
    -
    -			str = comma;
    +			/* Should we error out instead? */
     			continue;
     		}
     
    -		*bracket = 0;
    -		net_id = libcfs_str2net(strim(str));
    -		if (net_id == LNET_NIDNET(LNET_NID_ANY)) {
    -			tmp = str;
    -			goto failed_syntax;
    -		}
    +		/*
    +		 * All network paramaters are now known.
    +		 */
    +		nnets++;
     
     		/* always allocate a net, since we will eventually add an
     		 * interface to it, or we will fail, in which case we'll
    @@ -562,88 +634,107 @@ lnet_parse_networks(struct list_head *netlist, char *networks)
     		if (IS_ERR_OR_NULL(net))
     			goto failed;
     
    -		ni = lnet_ni_alloc(net, el, NULL);
    -		if (IS_ERR_OR_NULL(ni))
    -			goto failed;
    -
    -		if (el) {
    -			cfs_expr_list_free(el);
    -			el = NULL;
    -		}
    -
    -		niface = 0;
    -		iface = bracket + 1;
    +		if (!nistr) {
    +			/*
    +			 * No interface list was specified, allocate a
    +			 * ni using the defaults.
    +			 */
    +			ni = lnet_ni_alloc(net, net_el, NULL);
    +			if (IS_ERR_OR_NULL(ni))
    +				goto failed;
     
    -		bracket = strchr(iface, ')');
    -		if (!bracket) {
    -			tmp = iface;
    -			goto failed_syntax;
    +			if (net_el) {
    +				cfs_expr_list_free(net_el);
    +				net_el = NULL;
    +			}
    +			continue;
     		}
     
    -		*bracket = 0;
     		do {
    -			comma = strchr(iface, ',');
    -			if (comma)
    -				*comma++ = 0;
    -
    -			iface = strim(iface);
    -			if (!*iface) {
    -				tmp = iface;
    -				goto failed_syntax;
    +			elstr = NULL;
    +
    +			/* Interface name (mandatory) */
    +			while (isspace(*nistr))
    +				*nistr++ = '\0';
    +			name = nistr;
    +			nistr += strcspn(nistr, SPACESTR "[],");
    +			while (isspace(*nistr))
    +				*nistr++ = '\0';
    +
    +			/* CPT expression (optional) */
    +			if (*nistr == '[') {
    +				elstr = nistr;
    +				nistr += strcspn(nistr, "]");
    +				if (*nistr != ']') {
    +					str = elstr;
    +					goto failed_syntax;
    +				}
    +				rc = cfs_expr_list_parse(elstr,
    +							nistr - elstr + 1,
    +							0, LNET_CPT_NUMBER - 1,
    +							&ni_el);
    +				if (rc != 0) {
    +					str = elstr;
    +					goto failed_syntax;
    +				}
    +				*elstr = '\0';
    +				do {
    +					*nistr++ = '\0';
    +				} while (isspace(*nistr));
    +			} else {
    +				ni_el = net_el;
     			}
     
    -			if (niface == LNET_MAX_INTERFACES) {
    -				LCONSOLE_ERROR_MSG(0x115,
    -						   "Too many interfaces for net %s\n",
    -						   libcfs_net2str(net_id));
    -				goto failed;
    +			/*
    +			 * End of single interface specificaton,
    +			 * advance to the start of the next one, if
    +			 * any.
    +			 */
    +			if (*nistr == ',') {
    +				do {
    +					*nistr++ = '\0';
    +				} while (isspace(*nistr));
    +				if (!*nistr) {
    +					str = nistr;
    +					goto failed_syntax;
    +				}
    +			} else if (*nistr) {
    +				str = nistr;
    +				goto failed_syntax;
     			}
     
     			/*
    -			 * Allocate a separate piece of memory and copy
    -			 * into it the string, so we don't have
    -			 * a depencency on the tokens string.  This way we
    -			 * can free the tokens@the end of the function.
    -			 * The newly allocated ni_interfaces[] can be
    -			 * freed when freeing the NI
    +			 * At this point the name
    +			 is properly terminated.
     			 */
    -			ni->ni_interfaces[niface] = kstrdup(iface, GFP_KERNEL);
    -			if (!ni->ni_interfaces[niface]) {
    -				CERROR("Can't allocate net interface name\n");
    -				goto failed;
    -			}
    -			niface++;
    -			iface = comma;
    -		} while (iface);
    -
    -		str = bracket + 1;
    -		comma = strchr(bracket + 1, ',');
    -		if (comma) {
    -			*comma = 0;
    -			str = strim(str);
    -			if (*str) {
    -				tmp = str;
    +			if (!*name) {
    +				str = name;
     				goto failed_syntax;
     			}
    -			str = comma + 1;
    -			continue;
    -		}
     
    -		str = strim(str);
    -		if (*str) {
    -			tmp = str;
    -			goto failed_syntax;
    -		}
    -	}
    +			ni = lnet_ni_alloc(net, ni_el, name);
    +			if (IS_ERR_OR_NULL(ni))
    +				goto failed;
     
    -	list_for_each(temp_node, netlist)
    -		nnets++;
    +			if (ni_el) {
    +				if (ni_el != net_el) {
    +					cfs_expr_list_free(ni_el);
    +					ni_el = NULL;
    +				}
    +			}
    +		} while (*nistr);
    +
    +		if (net_el) {
    +			cfs_expr_list_free(net_el);
    +			net_el = NULL;
    +		}
    +	} while (*str);
     
     	kfree(tokens);
     	return nnets;
     
      failed_syntax:
    -	lnet_syntax("networks", networks, (int)(tmp - tokens), strlen(tmp));
    +	lnet_syntax("networks", networks, (int)(str - tokens), strlen(str));
      failed:
     	/* free the net list and all the nis on each net */
     	while (!list_empty(netlist)) {
    @@ -653,8 +744,10 @@ lnet_parse_networks(struct list_head *netlist, char *networks)
     		lnet_net_free(net);
     	}
     
    -	if (el)
    -		cfs_expr_list_free(el);
    +	if (ni_el && ni_el != net_el)
    +		cfs_expr_list_free(ni_el);
    +	if (net_el)
    +		cfs_expr_list_free(net_el);
     
     	kfree(tokens);
author
#93 lore raw

PATCH 34/34 lnet: introduce use_tcp_bonding mod param

In reply to NeilBrown

This is part of
8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
LU-7734 lnet: Multi-Rail local NI split

---
 .../staging/lustre/include/linux/lnet/lib-lnet.h   |    3 +
 drivers/staging/lustre/lnet/lnet/api-ni.c          |   22 ++++++++-
 drivers/staging/lustre/lnet/lnet/config.c          |   50 ++++++++++++++++----
 3 files changed, 61 insertions(+), 14 deletions(-)
drivers/staging/lustre/include/linux/lnet/lib-lnet.h +2 -1
index ef551b571935..5ee770cd7a5f 100644
@@ -629,7 +629,8 @@ void lnet_swap_pinginfo(struct lnet_ping_info *info);
 
 int lnet_parse_ip2nets(char **networksp, char *ip2nets);
 int lnet_parse_routes(char *route_str, int *im_a_router);
-int lnet_parse_networks(struct list_head *nilist, char *networks);
+int lnet_parse_networks(struct list_head *nilist, char *networks,
+			bool use_tcp_bonding);
 bool lnet_net_unique(__u32 net_id, struct list_head *nilist,
 		     struct lnet_net **net);
 bool lnet_ni_unique_net(struct list_head *nilist, char *iface);
drivers/staging/lustre/lnet/lnet/api-ni.c +20 -2
index ff5149da2d79..8ff386992c99 100644
@@ -59,6 +59,11 @@ static int rnet_htable_size = LNET_REMOTE_NETS_HASH_DEFAULT;
 module_param(rnet_htable_size, int, 0444);
 MODULE_PARM_DESC(rnet_htable_size, "size of remote network hash table");
 
+static int use_tcp_bonding = false;
+module_param(use_tcp_bonding, int, 0444);
+MODULE_PARM_DESC(use_tcp_bonding,
+		 "Set to 1 to use socklnd bonding. 0 to use Multi-Rail");
+
 static int lnet_ping(struct lnet_process_id id, signed long timeout,
 		     struct lnet_process_id __user *ids, int n_ids);
 
@@ -1446,6 +1451,18 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
 	 * to avoid a memory leak.
 	 */
 
+	/*
+	 * When a network uses TCP bonding then all its interfaces
+	 * must be specified when the network is first defined: the
+	 * TCP bonding code doesn't allow for interfaces to be added
+	 * or removed.
+	 */
+	if (net_l != net && net_l != NULL && use_tcp_bonding &&
+	    LNET_NETTYP(net_l->net_id) == SOCKLND) {
+		rc = -EINVAL;
+		goto failed0;
+	}
+
 	while (!list_empty(&net->net_ni_added)) {
 		ni = list_entry(net->net_ni_added.next, struct lnet_ni,
 				ni_netlist);
@@ -1702,7 +1719,8 @@ LNetNIInit(lnet_pid_t requested_pid)
 	 * routes if it has been loaded
 	 */
 	if (!the_lnet.ln_nis_from_mod_params) {
-		rc = lnet_parse_networks(&net_head, lnet_get_networks());
+		rc = lnet_parse_networks(&net_head, lnet_get_networks(),
+					 use_tcp_bonding);
 		if (rc < 0)
 			goto err_empty_list;
 	}
@@ -2000,7 +2018,7 @@ lnet_dyn_add_ni(lnet_pid_t requested_pid, struct lnet_ioctl_config_data *conf)
 		lnd_tunables = (struct lnet_ioctl_config_lnd_tunables *)conf->cfg_bulk;
 
 	/* Create a net/ni structures for the network string */
-	rc = lnet_parse_networks(&net_head, nets);
+	rc = lnet_parse_networks(&net_head, nets, use_tcp_bonding);
 	if (rc <= 0)
 		return !rc ? -EINVAL : rc;
 
drivers/staging/lustre/lnet/lnet/config.c +39 -11

93 lines not on this page · raw message

#94 lore raw

PATCH 34/34 lnet: introduce use_tcp_bonding mod param

Doug

?On 9/6/18, 5:56 PM, "NeilBrown" <[email protected]> wrote:

    This is part of
        8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
           LU-7734 lnet: Multi-Rail local NI split
    
    Signed-off-by: NeilBrown <[email protected]>
    ---
     .../staging/lustre/include/linux/lnet/lib-lnet.h   |    3 +
     drivers/staging/lustre/lnet/lnet/api-ni.c          |   22 ++++++++-
     drivers/staging/lustre/lnet/lnet/config.c          |   50 ++++++++++++++++----
     3 files changed, 61 insertions(+), 14 deletions(-)
    
    diff --git a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
    index ef551b571935..5ee770cd7a5f 100644
    --- a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
    +++ b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
    @@ -629,7 +629,8 @@ void lnet_swap_pinginfo(struct lnet_ping_info *info);
     
     int lnet_parse_ip2nets(char **networksp, char *ip2nets);
     int lnet_parse_routes(char *route_str, int *im_a_router);
    -int lnet_parse_networks(struct list_head *nilist, char *networks);
    +int lnet_parse_networks(struct list_head *nilist, char *networks,
    +			bool use_tcp_bonding);
     bool lnet_net_unique(__u32 net_id, struct list_head *nilist,
     		     struct lnet_net **net);
     bool lnet_ni_unique_net(struct list_head *nilist, char *iface);
    diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
    index ff5149da2d79..8ff386992c99 100644
    --- a/drivers/staging/lustre/lnet/lnet/api-ni.c
    +++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
    @@ -59,6 +59,11 @@ static int rnet_htable_size = LNET_REMOTE_NETS_HASH_DEFAULT;
     module_param(rnet_htable_size, int, 0444);
     MODULE_PARM_DESC(rnet_htable_size, "size of remote network hash table");
     
    +static int use_tcp_bonding = false;
    +module_param(use_tcp_bonding, int, 0444);
    +MODULE_PARM_DESC(use_tcp_bonding,
    +		 "Set to 1 to use socklnd bonding. 0 to use Multi-Rail");
    +
     static int lnet_ping(struct lnet_process_id id, signed long timeout,
     		     struct lnet_process_id __user *ids, int n_ids);
     
    @@ -1446,6 +1451,18 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
     	 * to avoid a memory leak.
     	 */
     
    +	/*
    +	 * When a network uses TCP bonding then all its interfaces
    +	 * must be specified when the network is first defined: the
    +	 * TCP bonding code doesn't allow for interfaces to be added
    +	 * or removed.
    +	 */
    +	if (net_l != net && net_l != NULL && use_tcp_bonding &&
    +	    LNET_NETTYP(net_l->net_id) == SOCKLND) {
    +		rc = -EINVAL;
    +		goto failed0;
    +	}
    +
     	while (!list_empty(&net->net_ni_added)) {
     		ni = list_entry(net->net_ni_added.next, struct lnet_ni,
     				ni_netlist);
    @@ -1702,7 +1719,8 @@ LNetNIInit(lnet_pid_t requested_pid)
     	 * routes if it has been loaded
     	 */
     	if (!the_lnet.ln_nis_from_mod_params) {
    -		rc = lnet_parse_networks(&net_head, lnet_get_networks());
    +		rc = lnet_parse_networks(&net_head, lnet_get_networks(),
    +					 use_tcp_bonding);
     		if (rc < 0)
     			goto err_empty_list;
     	}
    @@ -2000,7 +2018,7 @@ lnet_dyn_add_ni(lnet_pid_t requested_pid, struct lnet_ioctl_config_data *conf)
     		lnd_tunables = (struct lnet_ioctl_config_lnd_tunables *)conf->cfg_bulk;
     
     	/* Create a net/ni structures for the network string */
    -	rc = lnet_parse_networks(&net_head, nets);
    +	rc = lnet_parse_networks(&net_head, nets, use_tcp_bonding);
     	if (rc <= 0)
     		return !rc ? -EINVAL : rc;
     
    diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c
    index 0571fa6a7249..abfc5d8dc219 100644
    --- a/drivers/staging/lustre/lnet/lnet/config.c
    +++ b/drivers/staging/lustre/lnet/lnet/config.c
    @@ -117,6 +117,21 @@ lnet_ni_unique_net(struct list_head *nilist, char *iface)
     	return true;
     }
     
    +/* check that the NI is unique to the interfaces with in the same NI.
    + * This is only a consideration if use_tcp_bonding is set */
    +static bool
    +lnet_ni_unique_ni(char *iface_list[LNET_MAX_INTERFACES], char *iface)
    +{
    +	int i;
    +	for (i = 0; i < LNET_MAX_INTERFACES; i++) {
    +		if (iface_list[i] != NULL &&
    +		    strncmp(iface_list[i], iface, strlen(iface)) == 0)
    +			return false;
    +	}
    +
    +	return true;
    +}
    +
     static bool
     in_array(__u32 *array, __u32 size, __u32 value)
     {
    @@ -374,6 +389,9 @@ lnet_ni_add_interface(struct lnet_ni *ni, char *iface)
     	if (ni == NULL)
     		return -ENOMEM;
     
    +	if (!lnet_ni_unique_ni(ni->ni_interfaces, iface))
    +		return -EINVAL;
    +
     	/* Allocate a separate piece of memory and copy
     	 * into it the string, so we don't have
     	 * a depencency on the tokens string.  This way we
    @@ -495,7 +513,8 @@ lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface)
      * nilist.
      */
     int
    -lnet_parse_networks(struct list_head *netlist, char *networks)
    +lnet_parse_networks(struct list_head *netlist, char *networks,
    +		    bool use_tcp_bonding)
     {
     	struct cfs_expr_list *net_el = NULL;
     	struct cfs_expr_list *ni_el = NULL;
    @@ -634,7 +653,8 @@ lnet_parse_networks(struct list_head *netlist, char *networks)
     		if (IS_ERR_OR_NULL(net))
     			goto failed;
     
    -		if (!nistr) {
    +		if (!nistr ||
    +		    (use_tcp_bonding && LNET_NETTYP(net_id) == SOCKLND)) {
     			/*
     			 * No interface list was specified, allocate a
     			 * ni using the defaults.
    @@ -643,11 +663,13 @@ lnet_parse_networks(struct list_head *netlist, char *networks)
     			if (IS_ERR_OR_NULL(ni))
     				goto failed;
     
    -			if (net_el) {
    -				cfs_expr_list_free(net_el);
    -				net_el = NULL;
    +			if (!nistr) {
    +				if (net_el) {
    +					cfs_expr_list_free(net_el);
    +					net_el = NULL;
    +				}
    +				continue;
     			}
    -			continue;
     		}
     
     		do {
    @@ -704,17 +726,23 @@ lnet_parse_networks(struct list_head *netlist, char *networks)
     			}
     
     			/*
    -			 * At this point the name
    -			 is properly terminated.
    +			 * At this point the name is properly terminated.
     			 */
     			if (!*name) {
     				str = name;
     				goto failed_syntax;
     			}
     
    -			ni = lnet_ni_alloc(net, ni_el, name);
    -			if (IS_ERR_OR_NULL(ni))
    -				goto failed;
    +			if (use_tcp_bonding &&
    +			    LNET_NETTYP(net->net_id) == SOCKLND) {
    +				rc = lnet_ni_add_interface(ni, name);
    +				if (rc != 0)
    +					goto failed;
    +			} else {
    +				ni = lnet_ni_alloc(net, ni_el, name);
    +				if (IS_ERR_OR_NULL(ni))
    +					goto failed;
    +			}
     
     			if (ni_el) {
     				if (ni_el != net_el) {
#95 lore raw

cover Beginning of multi-rail support for drivers/staging/lustre

In reply to NeilBrown
Quoted 22 lines

The following series implements the first patch in the multi-rail series:
Commit: 8cbb8cd3e771 ("LU-7734 lnet: Multi-Rail local NI split")

I split that commit up into 40 individual commits which can be found at
https://github.com/neilbrown/lustre/commits/multirail
though you need to scroll down a bit, as that contains all the multi-rail series.

I then ported most of these patches to my mainline tree. Some that I haven't included are:
- lnet: Move lnet_msg_alloc/free down a bit. lnet_msg_alloc/free don't exist any more
- lnet: lib-types: change some tabs to spaces
- lnet - assorted whitespace changes.
- lnet: change ni_last_alive from time64_t to long
- lnet: add lnet_net_state
net_state is never used.
- lnet: remove 'static' from lnet_get_net_config()

Quoted 9 lines

I've also made a couple of minor changes to individual patches not strictly related to porting (the net_prio field is never used, so I never added it - I should have made that a separate patch).

This series compiles, but doesn't work. I get a NULL pointer reference, then an assertion failure. If I fix those, it hangs. The NULL pointer ref and the failing assertion are gone with later patches, so I hope the other problems are too.

I have tried it and did a compare to what landed in the OpenSFS branch. I saw the failures in my testing and foudn the mistake in the 7th patch.

Quoted 4 lines

Some of these patches have very poor descriptions, such as "I have no idea what this does". If someone would like to explain - or maybe say "Oh, we really shouldn't have done that", I'd be very happy to receive that, and update the description or patch accordingly.

When I ran checkpatch it really dislikes:

This is part of
8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
LU-7734 lnet: Multi-Rail local NI split

I don't recommend landing the above in the commit messsage as for the reason that a person outside of lustre will not know where to look for that git commit. Instead I recommend replacing it with:

------------------------------------------------------------------

WC-bug-id: https://jira.whamcloud.com/browse/LU-7734
Reviewed-on: http://review.whamcloud.com/18274

This gives the reviewer a URL link for both the JIRA ticket that usually contains details not in the commit message as well as the gerrit URL for the original patch. This way if a future bug is found a comparison can be done against the original patch.

The policy for the Lustre project is to perserve authorship for patches when porting to other branches, upstream or LTS.

Quoted 78 lines

These will all appear in my lustre-testing branch, but won't migrate to 'lustre' until I, at least, have enough other patches that I can get a successful test run.

Review and comments always welcome.

Thanks,
NeilBrown

---

Amir Shehata (1):
Completely re-write lnet_parse_networks().

NeilBrown (33):
struct lnet_ni - reformat comments.
lnet: Create struct lnet_net
lnet: struct lnet_ni: move ni_lnd to lnet_net
lnet: embed lnd_tunables in lnet_ni
lnet: begin separating "networks" from "network interfaces".
lnet: store separate xmit/recv net-interface in each message.
lnet: change lnet_peer to reference the net, rather than ni.
lnet: add cpt to lnet_match_info.
lnet: add list of cpts to lnet_net.
lnet: add ni arg to lnet_cpt_of_nid()
lnet: pass tun to lnet_startup_lndni, instead of full conf
lnet: split lnet_startup_lndni
lnet: reverse order of lnet_startup_lnd{net,ni}
lnet: rename lnet_find_net_locked to lnet_find_rnet_locked
lnet: extend zombie handling to nets and nis
lnet: lnet_shutdown_lndnets - remove some cleanup code.
lnet: move lnet_shutdown_lndnets down to after first use
lnet: add ni_state
lnet: simplify lnet_islocalnet()
lnet: discard ni_cpt_list
lnet: add net_ni_added
lnet: don't take reference in lnet_XX2ni_locked()
lnet: don't need lock to test ln_shutdown.
lnet: don't take lock over lnet_net_unique()
lnet: swap 'then' and 'else' branches in lnet_startup_lndnet
lnet: only valid lnd_type when net_id is unique.
lnet: make it possible to add a new interface to a network
lnet: add checks to ensure network interface names are unique.
lnet: track tunables in lnet_startup_lndnet()
lnet: fix typo
lnet: lnet_dyn_add_ni: fix ping_info count
lnet: lnet_dyn_del_ni: fix ping_info count
lnet: introduce use_tcp_bonding mod param

 .../staging/lustre/include/linux/lnet/lib-lnet.h   |   31 -
 .../staging/lustre/include/linux/lnet/lib-types.h  |  142 ++-
 .../lustre/include/uapi/linux/lnet/lnet-dlc.h      |   18 
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c    |   10 
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h    |    6 
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c |   12 
 .../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c  |   74 +-
 .../staging/lustre/lnet/klnds/socklnd/socklnd.c    |   25 -
 drivers/staging/lustre/lnet/lnet/acceptor.c        |    8 
 drivers/staging/lustre/lnet/lnet/api-ni.c          |  939 +++++++++++++-------
 drivers/staging/lustre/lnet/lnet/config.c          |  688 +++++++++++----
 drivers/staging/lustre/lnet/lnet/lib-move.c        |  132 ++-
 drivers/staging/lustre/lnet/lnet/lib-ptl.c         |    6 
 drivers/staging/lustre/lnet/lnet/lo.c              |    2 
 drivers/staging/lustre/lnet/lnet/net_fault.c       |    3 
 drivers/staging/lustre/lnet/lnet/peer.c            |   31 -
 drivers/staging/lustre/lnet/lnet/router.c          |   51 +
 drivers/staging/lustre/lnet/lnet/router_proc.c     |   24 -
 drivers/staging/lustre/lnet/selftest/brw_test.c    |    2 
 drivers/staging/lustre/lnet/selftest/framework.c   |    3 
 drivers/staging/lustre/lnet/selftest/selftest.h    |    2 
 21 files changed, 1507 insertions(+), 702 deletions(-)

--
Signature

author
#96 lore raw

cover Beginning of multi-rail support for drivers/staging/lustre

Quoted 49 lines · James Simmons
Quoted 21 lines

The following series implements the first patch in the multi-rail series:
Commit: 8cbb8cd3e771 ("LU-7734 lnet: Multi-Rail local NI split")

I split that commit up into 40 individual commits which can be found at
https://github.com/neilbrown/lustre/commits/multirail
though you need to scroll down a bit, as that contains all the multi-rail series.

I then ported most of these patches to my mainline tree. Some that I haven't included are:
- lnet: Move lnet_msg_alloc/free down a bit. lnet_msg_alloc/free don't exist any more
- lnet: lib-types: change some tabs to spaces
- lnet - assorted whitespace changes.
- lnet: change ni_last_alive from time64_t to long
- lnet: add lnet_net_state
net_state is never used.
- lnet: remove 'static' from lnet_get_net_config()

Quoted 9 lines

I've also made a couple of minor changes to individual patches not strictly related to porting (the net_prio field is never used, so I never added it - I should have made that a separate patch).

This series compiles, but doesn't work. I get a NULL pointer reference, then an assertion failure. If I fix those, it hangs. The NULL pointer ref and the failing assertion are gone with later patches, so I hope the other problems are too.

I have tried it and did a compare to what landed in the OpenSFS branch. I saw the failures in my testing and foudn the mistake in the 7th patch.

Quoted 4 lines

Some of these patches have very poor descriptions, such as "I have no idea what this does". If someone would like to explain - or maybe say "Oh, we really shouldn't have done that", I'd be very happy to receive that, and update the description or patch accordingly.

When I ran checkpatch it really dislikes:

This is part of
8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
LU-7734 lnet: Multi-Rail local NI split

I don't recommend landing the above in the commit messsage as for the reason that a person outside of lustre will not know where to look for that git commit. Instead I recommend replacing it with:

Quoted 7 lines

------------------------------------------------------------------

WC-bug-id: https://jira.whamcloud.com/browse/LU-7734
Reviewed-on: http://review.whamcloud.com/18274

Thanks for the suggestion. I don't like that approach exactly because it seems to be a lie. The specific patch was not reviewed by those people, and there is useful information which is not included there. I have changed to patches to include:

This is part of
Commit: 8cbb8cd3e771 ("LU-7734 lnet: Multi-Rail local NI split")
from upstream lustre, where it is marked:

WC-bug-id: https://jira.whamcloud.com/browse/LU-7734
Reviewed-on: http://review.whamcloud.com/18274

checkpatch is not happy with the indented tags, but checkpatch is a servant, not the master.

I've made that change, moved this series to my 'lustre' branch, merged in the latest -rc, and pushed it all out.

Now the post the rest of the 'MR' series, and then start looking at Dynamic Discovery (756abb9cf00b9^..1c45d9051764)

Thanks,
NeilBrown

Quoted 87 lines

This gives the reviewer a URL link for both the JIRA ticket that usually contains details not in the commit message as well as the gerrit URL for the original patch. This way if a future bug is found a comparison can be done against the original patch.

The policy for the Lustre project is to perserve authorship for patches when porting to other branches, upstream or LTS.

Quoted 78 lines

These will all appear in my lustre-testing branch, but won't migrate to 'lustre' until I, at least, have enough other patches that I can get a successful test run.

Review and comments always welcome.

Thanks,
NeilBrown

---

Amir Shehata (1):
Completely re-write lnet_parse_networks().

NeilBrown (33):
struct lnet_ni - reformat comments.
lnet: Create struct lnet_net
lnet: struct lnet_ni: move ni_lnd to lnet_net
lnet: embed lnd_tunables in lnet_ni
lnet: begin separating "networks" from "network interfaces".
lnet: store separate xmit/recv net-interface in each message.
lnet: change lnet_peer to reference the net, rather than ni.
lnet: add cpt to lnet_match_info.
lnet: add list of cpts to lnet_net.
lnet: add ni arg to lnet_cpt_of_nid()
lnet: pass tun to lnet_startup_lndni, instead of full conf
lnet: split lnet_startup_lndni
lnet: reverse order of lnet_startup_lnd{net,ni}
lnet: rename lnet_find_net_locked to lnet_find_rnet_locked
lnet: extend zombie handling to nets and nis
lnet: lnet_shutdown_lndnets - remove some cleanup code.
lnet: move lnet_shutdown_lndnets down to after first use
lnet: add ni_state
lnet: simplify lnet_islocalnet()
lnet: discard ni_cpt_list
lnet: add net_ni_added
lnet: don't take reference in lnet_XX2ni_locked()
lnet: don't need lock to test ln_shutdown.
lnet: don't take lock over lnet_net_unique()
lnet: swap 'then' and 'else' branches in lnet_startup_lndnet
lnet: only valid lnd_type when net_id is unique.
lnet: make it possible to add a new interface to a network
lnet: add checks to ensure network interface names are unique.
lnet: track tunables in lnet_startup_lndnet()
lnet: fix typo
lnet: lnet_dyn_add_ni: fix ping_info count
lnet: lnet_dyn_del_ni: fix ping_info count
lnet: introduce use_tcp_bonding mod param

 .../staging/lustre/include/linux/lnet/lib-lnet.h   |   31 -
 .../staging/lustre/include/linux/lnet/lib-types.h  |  142 ++-
 .../lustre/include/uapi/linux/lnet/lnet-dlc.h      |   18 
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c    |   10 
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h    |    6 
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c |   12 
 .../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c  |   74 +-
 .../staging/lustre/lnet/klnds/socklnd/socklnd.c    |   25 -
 drivers/staging/lustre/lnet/lnet/acceptor.c        |    8 
 drivers/staging/lustre/lnet/lnet/api-ni.c          |  939 +++++++++++++-------
 drivers/staging/lustre/lnet/lnet/config.c          |  688 +++++++++++----
 drivers/staging/lustre/lnet/lnet/lib-move.c        |  132 ++-
 drivers/staging/lustre/lnet/lnet/lib-ptl.c         |    6 
 drivers/staging/lustre/lnet/lnet/lo.c              |    2 
 drivers/staging/lustre/lnet/lnet/net_fault.c       |    3 
 drivers/staging/lustre/lnet/lnet/peer.c            |   31 -
 drivers/staging/lustre/lnet/lnet/router.c          |   51 +
 drivers/staging/lustre/lnet/lnet/router_proc.c     |   24 -
 drivers/staging/lustre/lnet/selftest/brw_test.c    |    2 
 drivers/staging/lustre/lnet/selftest/framework.c   |    3 
 drivers/staging/lustre/lnet/selftest/selftest.h    |    2 
 21 files changed, 1507 insertions(+), 702 deletions(-)

--
Signature

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 832 bytes
Desc: not available
URL: <http://lists.lustre.org/pipermail/lustre-devel-lustre.org/attachments/20180924/ce3ab1c5/attachment.sig>

#97 lore raw

cover Beginning of multi-rail support for drivers/staging/lustre

Quoted 16 lines
Quoted 7 lines

WC-bug-id: https://jira.whamcloud.com/browse/LU-7734
Reviewed-on: http://review.whamcloud.com/18274

Thanks for the suggestion. I don't like that approach exactly because it seems to be a lie. The specific patch was not reviewed by those people, and there is useful information which is not included there. I have changed to patches to include:

This is part of
Commit: 8cbb8cd3e771 ("LU-7734 lnet: Multi-Rail local NI split")
from upstream lustre, where it is marked:

Quoted 8 lines

WC-bug-id: https://jira.whamcloud.com/browse/LU-7734
Reviewed-on: http://review.whamcloud.com/18274

checkpatch is not happy with the indented tags, but checkpatch is a servant, not the master.

To my knowledge their isn't really a policy about this. What I have been doing is kind of following how LTS versions of lustre have been handled. For LTS versions patches are cherry-picked and two additional lines are added:

Lustre-change:
Lustre-commit:

The orginal reviews are keep. Also by including the original reviews the people involved with those patches are poked. The only requirement is that 2 people review again. Not everyone has to review for it to land. Once landed I don't see a clear why to tell who reviewed.

In any case the above approach seems reasonable as long as the original author is preserve. The general rule is the original patch poster normally keeps authorship. Also I noticed patches recently pushed are not reaching the original authors and reviewers. We should make sure that still happens.