Skip to contentSkip to conversation

Add support for preloading IPv4 and IPv6 ACM caches

PATCH In review Hal Rosenstock 2 participants 3 messages Jun 27, 2013 – Jun 28, 2013 2 messages not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH v2 15/16 Add support for preloading IPv4 and IPv6 ACM caches

A hosts file (ibacm_hosts.cfg) is configured with tuples of IP addresses (either IPv4 or IPv6) and IB GID. This information in conjunction with the path record preloading allows for the IPv4 and IPv6 ACM caches to be preloaded.

IPv4 was tested with both rping and ucmatose as well as acme. IPv6 was tested with ucmatose and acme.

The remaining ACM cache to be preloaded is the name cache. This will be added in subsequent patch.

---
Change from v1:
Eliminated command line option in favor of option file parameters

 Makefile.am     |    2 +-
 acm_notes.txt   |    3 ++
 ibacm_hosts.cfg |   12 ++++++++
 ibacm_opts.cfg  |    6 ++++
 man/ibacm.1     |    4 ++
 src/acm.c       |   83 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/acme.c      |    6 ++++
 7 files changed, 115 insertions(+), 1 deletions(-)
 create mode 100644 ibacm_hosts.cfg
Makefile.am +1 -1
index de035bc..c97326a 100644
@@ -20,7 +20,7 @@ man_MANS = \
 
 EXTRA_DIST = src/acm_mad.h src/libacm.h ibacm.init.in \
 	     linux/osd.h linux/dlist.h ibacm.spec.in $(man_MANS) ibacm_opts.cfg \
-	     ibacm_addr.cfg
+	     ibacm_addr.cfg ibacm_hosts.cfg
 
 install-exec-hook:
 	install -D -m 755 ibacm.init $(DESTDIR)$(sysconfdir)/init.d/ibacm;
acm_notes.txt +3 -0
index 50f0286..991b251 100644
@@ -138,3 +138,6 @@ none which does not preload these caches.  See dump_pr.notes.txt in dump_pr
 for more information on the full_opensm_v1 file format and how to configure
 OpenSM to generate this file.
 
+Additional IPv4 and IPv6 caches can be be preloaded by using the
+hosts_file option.  The default is "none" which does not preload these
+caches.
ibacm_hosts.cfg new file +12 -0
new file mode 100644
index 0000000..32b6594
@@ -0,0 +1,12 @@
+# InfiniBand Communication Management Assistant for clusters hosts file
+#
+# Entry format is:
+# address IB GID
+#
+# address - IPv4 or IPv6 formatted address
+#
+# There can be multiple entries for a single IB GID
+#
+# Samples:
+# 192.168.1.3 fe80::8:f104:39a:169 
+# fe80::208:f104:39a:169 fe80::8:f104:39a:169
ibacm_opts.cfg +6 -0
index 968a99d..0e21d37 100644
@@ -155,3 +155,9 @@ path_rec_fmt none
 # If path_rec_fmt is other than "none", full pathname of path records file
 # to use for preloading the ACM cache.  Default is ACM_CONF_DIR/ibacm_path_records.dump
 
+# hosts_file:
+# none - No hosts file
+# If hosts_file is other than "none", full pathname of hosts file
+
+hosts_file none
+
man/ibacm.1 +4 -0
index 7c5f83d..7e11393 100644
@@ -156,5 +156,9 @@ format.  Default format is none which does not preload these caches.
 See dump_pr.notes.txt in dump_pr for more information on the
 full_opensm_v1 file format and how to configure OpenSM to
 generate this file.
+.P
+Additional IPv4 and IPv6 caches can be be preloaded by using the
+hosts_file option.  The default is "none" which does not preload these
+caches.
 .SH "SEE ALSO"
 ibacm(7), ib_acme(1), rdma_cm(7)
src/acm.c +83 -0

119 lines not on this page · raw message

src/acme.c +6 -0
index 5ba5c0a..648e97f 100644
@@ -256,6 +256,12 @@ static void gen_opts_temp(FILE *f)
 	fprintf(f, "# If path_rec_fmt is other than \"none\", full pathname of path records file\n");
 	fprintf(f, "# to use for preloading the ACM cache.  Default is ACM_CONF_DIR/ibacm_path_records.dump\n");
 	fprintf(f, "\n");
+	fprintf(f, "# hosts_file:\n");
+	fprintf(f, "# none - No hosts file\n");
+	fprintf(f, "# If hosts_file is other than \"none\", full pathname of hosts file\n");
+	fprintf(f, "\n");
+	fprintf(f, "hosts_file none\n");
+	fprintf(f, "\n");
 }
 
 static int open_dir(void)
Signature · 6 lines
1.7.8.2

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): 51CC8F18.3000509-LDSdmyG8hGV8YrgS2mwiifqBs+8SCbDb@public.gmane.org

#2 lore raw
Quoted 33 lines
src/acm.c +33 -0
+static void acm_parse_hosts_file(struct acm_ep *ep)
+{
+	FILE *f;
+	char s[120];
+	char addr[INET6_ADDRSTRLEN], gid[INET6_ADDRSTRLEN];
+	uint8_t name[ACM_MAX_ADDRESS];
+	struct in6_addr ip_addr, ib_addr;
+	struct acm_dest *dest, *new_dest;
+	uint8_t addr_type;
+
+	if (!(f = fopen(hosts_file, "r"))) {
+		acm_log(0, "ERROR - couldn't open %s\n", hosts_file);
+		return;
+        }
+
+	while (fgets(s, sizeof s, f)) {
+		if (s[0] == '#')
+			continue;
+
+		if (sscanf(s, "%46s%46s", addr, gid) != 2)
+			continue;
+
+		acm_log(2, "%s", s);
+		if (inet_pton(AF_INET6, gid, &ib_addr) <= 0) {
+			acm_log(0, "ERROR - %s is not IB GID\n", gid);
+			continue;
+		}
+		if (inet_pton(AF_INET, addr, &ip_addr) > 0)
+			addr_type = ACM_ADDRESS_IP;
+		else if (inet_pton(AF_INET6, addr, &ip_addr) > 0)
+			addr_type = ACM_ADDRESS_IP6;
+		else {
+			acm_log(0, "ERROR - %s is not IP address\n", addr);

src/acm.c err: · line 2837

Quoted 10 lines
src/acm.c +10 -0
+			continue;
+		}
+
+		memset(name, 0, ACM_MAX_ADDRESS);
+		memcpy(name, &ib_addr, sizeof(ib_addr));
+		dest = acm_get_dest(ep, ACM_ADDRESS_GID, name);
+		if (!dest) {
+			acm_log(0, "ERROR - IB GID %s not found in cache\n", gid);
+			continue;
+		}

I just noticed that there's no matching acm_put_dest() to this call. I'll fix this up.

Quoted 6 lines
src/acm.c +6 -0
+
+		memset(name, 0, ACM_MAX_ADDRESS);
+		if (addr_type == ACM_ADDRESS_IP)
+			memcpy(name, &ip_addr, 4);
+		else
+			memcpy(name, &ip_addr, sizeof(ip_addr));

src/acm.c err: · line 2853

Quoted 10 lines
src/acm.c +10 -0
+		new_dest = acm_acquire_dest(ep, addr_type, name);
+		if (!new_dest) {
+			acm_log(0, "ERROR - unable to create dest %s\n", addr);
+			continue;
+		}
+		new_dest->path = dest->path;
+		new_dest->remote_qpn = dest->remote_qpn;
+		new_dest->addr_timeout = dest->addr_timeout;
+		new_dest->route_timeout = dest->route_timeout;
+		new_dest->state = dest->state;

Rather than skipping addresses that we can't lookup the GID for, I *think* it would work to simply add new_dest, but set the state to ACM_ADDR_RESOLVED.

I can quickly throw a patch together to this effect, but won't be able to test it immediately.

- Sean
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): 1828884A29C6694DAF28B7E6B8A823736FD378C2-P5GAC/[email protected]

#4 lore raw
Quoted 65 lines · Sean
Quoted 33 lines
src/acm.c +33 -0
+static void acm_parse_hosts_file(struct acm_ep *ep)
+{
+	FILE *f;
+	char s[120];
+	char addr[INET6_ADDRSTRLEN], gid[INET6_ADDRSTRLEN];
+	uint8_t name[ACM_MAX_ADDRESS];
+	struct in6_addr ip_addr, ib_addr;
+	struct acm_dest *dest, *new_dest;
+	uint8_t addr_type;
+
+	if (!(f = fopen(hosts_file, "r"))) {
+		acm_log(0, "ERROR - couldn't open %s\n", hosts_file);
+		return;
+        }
+
+	while (fgets(s, sizeof s, f)) {
+		if (s[0] == '#')
+			continue;
+
+		if (sscanf(s, "%46s%46s", addr, gid) != 2)
+			continue;
+
+		acm_log(2, "%s", s);
+		if (inet_pton(AF_INET6, gid, &ib_addr) <= 0) {
+			acm_log(0, "ERROR - %s is not IB GID\n", gid);
+			continue;
+		}
+		if (inet_pton(AF_INET, addr, &ip_addr) > 0)
+			addr_type = ACM_ADDRESS_IP;
+		else if (inet_pton(AF_INET6, addr, &ip_addr) > 0)
+			addr_type = ACM_ADDRESS_IP6;
+		else {
+			acm_log(0, "ERROR - %s is not IP address\n", addr);

src/acm.c err: · line 2837

Quoted 10 lines
src/acm.c +10 -0
+			continue;
+		}
+
+		memset(name, 0, ACM_MAX_ADDRESS);
+		memcpy(name, &ib_addr, sizeof(ib_addr));
+		dest = acm_get_dest(ep, ACM_ADDRESS_GID, name);
+		if (!dest) {
+			acm_log(0, "ERROR - IB GID %s not found in cache\n", gid);
+			continue;
+		}

I just noticed that there's no matching acm_put_dest() to this call. I'll fix this up.

Quoted 6 lines
src/acm.c +6 -0
+
+		memset(name, 0, ACM_MAX_ADDRESS);
+		if (addr_type == ACM_ADDRESS_IP)
+			memcpy(name, &ip_addr, 4);
+		else
+			memcpy(name, &ip_addr, sizeof(ip_addr));

src/acm.c err: · line 2853

Quoted 10 lines
src/acm.c +10 -0
+		new_dest = acm_acquire_dest(ep, addr_type, name);
+		if (!new_dest) {
+			acm_log(0, "ERROR - unable to create dest %s\n", addr);
+			continue;
+		}
+		new_dest->path = dest->path;
+		new_dest->remote_qpn = dest->remote_qpn;
+		new_dest->addr_timeout = dest->addr_timeout;
+		new_dest->route_timeout = dest->route_timeout;
+		new_dest->state = dest->state;

Rather than skipping addresses that we can't lookup the GID for, I *think* it would work to simply add new_dest, but set the state to ACM_ADDR_RESOLVED.

Quoted 1 line

I can quickly throw a patch together to this effect, but won't be able to test it immediately.

I can test it too.

-- Hal

Quoted 3 lines

- Sean

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html