nbft: use the correct interfaces for NBFT connections
PATCH 1/1 nbft: use the correct interfaces for NBFT connections
The NBFT specifies which network interface should be used to connect to each namespace (using the HFI descriptor), but that is currently ignored when "nvme connect-all --nbft" is used. If there are namespaces on different interfaces, some namespaces can fail to connect, unless the IP tables were already set up to route these in the desired way when the interface isn't specified.
Add a bit of logic to look at the HFI descriptors in the NBFT and find the network interface to use when connecting each namespace.
--- libnvme/src/nvme/fabrics.c | 91 +++++++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 2 deletions(-)
libnvme/src/nvme/fabrics.c +89 -2
@@ -11,6 +11,7 @@#include <errno.h> #include <fcntl.h> #include <fnmatch.h> +#include <ifaddrs.h> #include <inttypes.h> #include <limits.h> #include <stdbool.h>
@@ -21,7 +22,9 @@#include <unistd.h> #include <arpa/inet.h> +#include <linux/if_ether.h> #include <net/if.h> +#include <netpacket/packet.h> #include <netdb.h> #include <sys/param.h> #include <sys/stat.h>
@@ -2820,6 +2823,76 @@ static int nbft_discovery(struct libnvme_global_ctx *ctx,return 0 ; } +#define VLAN_PROC_PATH "/proc/net/vlan" + +/* +* Return 0 for no vlan_id, to be consistent with the NBFT spec. +*/ +static int get_vlan_id (const char *ifname) +{ +char path[256 ], line[256 ]; +int vlan_id =0 ; +FILE *f; + +snprintf (path,sizeof (path),"%s/%s" ,VLAN_PROC_PATH , ifname); + f =fopen (path,"r" ); +if (!f) +return 0 ; + +while (fgets (line,sizeof (line), f)) { +if (sscanf (line," VID: %d" , &vlan_id) ==1 ) { +fclose (f); +return vlan_id; + } + } + +fclose (f); +return 0 ; +} + +/* +* Find network interface corresponding to the NBFT HFI +* by looking for mac address and vlan id. +*/ +static char *nbft_find_hfi_iface (struct libnbft_hfi *hfi) +{ +struct ifaddrs *ifaddr, *ifa; +char *result =NULL ; + +if (strcmp ((char *)hfi->transport,"tcp" )) +return NULL ; + +if (getifaddrs (&ifaddr) !=0 ) +return NULL ; + +for (ifa = ifaddr; ifa !=NULL ; ifa = ifa->ifa_next) { +struct sockaddr_ll *sll; + +if (!ifa->ifa_addr) +continue ; + +if (ifa->ifa_addr->sa_family !=AF_PACKET ) +continue ; + + sll = (struct sockaddr_ll *)ifa->ifa_addr; + +if (sll->sll_halen !=ETH_ALEN ) +continue ; + +if (!memcmp (sll->sll_addr, hfi->tcp_info.mac_addr,ETH_ALEN )) { +int vlan_id =get_vlan_id (ifa->ifa_name); + +if (vlan_id == hfi->tcp_info.vlan) { + result =strdup (ifa->ifa_name); +break ; + } + } + } + +freeifaddrs (ifaddr); +return result; +} + __public int libnvmf_discovery_nbft(struct libnvme_global_ctx *ctx,struct libnvmf_context *fctx,bool connect,char *nbft_path) {
@@ -2905,7 +2978,11 @@ __public int libnvmf_discovery_nbft(struct libnvme_global_ctx *ctx, nfctx.transport = (*ss)->transport; nfctx.traddr = (*ss)->traddr; nfctx.trsvcid = (*ss)->trsvcid; - nfctx.host_iface =NULL ; + nfctx.host_iface =nbft_find_hfi_iface (hfi); +if (!nfctx.host_iface) +libnvme_msg (ctx,LIBNVME_LOG_INFO , +"SSNS %d: could not find host interface for HFI %d\n" , + (*ss)->index, hfi->index); rr =nbft_connect (ctx, &nfctx, h,NULL , *ss);
@@ -2929,6 +3006,9 @@ __public int libnvmf_discovery_nbft(struct libnvme_global_ctx *ctx, host_traddr); } +if (nfctx.host_iface) +free ((char *)nfctx.host_iface); +if (rr) { libnvme_msg(ctx, LIBNVME_LOG_ERR, "SSNS %d: no controller found\n",
@@ -2993,7 +3073,11 @@ __public int libnvmf_discovery_nbft(struct libnvme_global_ctx *ctx, nfctx.traddr = uri->host; nfctx.trsvcid = trsvcid; nfctx.host_traddr = host_traddr; - nfctx.host_iface =NULL ; + nfctx.host_iface =nbft_find_hfi_iface (hfi); +if (!nfctx.host_iface) +libnvme_msg (ctx,LIBNVME_LOG_INFO , +"SSNS %d: could not find host interface for HFI %d\n" , + (*ss)->index, hfi->index);/* Lookup existing discovery controller */ c =lookup_ctrl (h, &nfctx);
@@ -3013,6 +3097,9 @@ __public int libnvmf_discovery_nbft(struct libnvme_global_ctx *ctx, }else ret =0 ; +if (nfctx.host_iface) +free ((char *)nfctx.host_iface); +if (ret) { libnvme_msg(ctx, LIBNVME_LOG_ERR, "Discovery Descriptor %d: failed to add discovery controller: %s\n",
Signature · 1 line
2.51.0
Quoted 11 lines · Stuart Hayes
The NBFT specifies which network interface should be used to connect to each namespace (using the HFI descriptor), but that is currently ignored when "nvme connect-all --nbft" is used. If there are namespaces on different interfaces, some namespaces can fail to connect, unless the IP tables were already set up to route these in the desired way when the interface isn't specified.
Add a bit of logic to look at the HFI descriptors in the NBFT and find the network interface to use when connecting each namespace.
Signed-off-by Stuart Hayes<[email protected]>
Should I submit this as a PR, rather than sending the patch to this list?
Thanks!
Keith Busch
maintainerQuoted 8 lines · stuart hayes
Quoted 7 lines · Stuart Hayes
The NBFT specifies which network interface should be used to connect to each namespace (using the HFI descriptor), but that is currently ignored when "nvme connect-all --nbft" is used. If there are namespaces on different interfaces, some namespaces can fail to connect, unless the IP tables were already set up to route these in the desired way when the interface isn't specified.
Quoted 6 lines
Quoted 4 lines
Add a bit of logic to look at the HFI descriptors in the NBFT and find the network interface to use when connecting each namespace.
Signed-off-by Stuart Hayes<[email protected]>Should I submit this as a PR, rather than sending the patch to this list?
Maybe the PR is a quicker route. The mailing list historically picked up these things too, but you normally need to alter the subject to [PATCH nvme-cli] or something like that so it stands out separately from the usual kernel patches.
Quoted 6 lines · Keith Busch
Quoted 1 line
Should I submit this as a PR, rather than sending the patch to this list?
Maybe the PR is a quicker route. The mailing list historically picked up these things too, but you normally need to alter the subject to [PATCH nvme-cli] or something like that so it stands out separately from the usual kernel patches.
Sorry, didn't see it because focusing a bit much on getting nvme-cli 3.x out the door and not scanning the mailing list closely. But this is only temporary, promised.
Daniel Wagner
AppliedQuoted 8 lines · Stuart Hayes
The NBFT specifies which network interface should be used to connect to each namespace (using the HFI descriptor), but that is currently ignored when "nvme connect-all --nbft" is used. If there are namespaces on different interfaces, some namespaces can fail to connect, unless the IP tables were already set up to route these in the desired way when the interface isn't specified.
[...]
Applied, thanks!
[1/1] nbft: use the correct interfaces for NBFT connections
commit: 685861ae563e3bcd77f2224d0206e448e426db01
Best regards,
Signature · 1 line
Daniel Wagner <[email protected]>