Skip to contentSkip to conversation

Add RDMA support to the pNFS file+flexfiles data channels

PATCH In review [email protected] 3 participants 23 messages Nov 10, 2020 – Dec 1, 2020

Subsystems (3)
lore
Share & tools
lore

[email protected]

author maintainer
#0 lore raw

v3 cover Add RDMA support to the pNFS file+flexfiles data channels

From: Trond Myklebust <[email protected]>

Add support for connecting to the pNFS files/flexfiles data servers through RDMA, assuming that the GETDEVICEINFO call advertises that support.

v2: Fix layoutstats encoding for pNFS/flexfiles.
v3: Move most of the netid handling into the SUNRPC and RDMA modules.
Fix up the mount code to benefit more from automated loading of SUNRPC transport modules.

Trond Myklebust (11):
SUNRPC: xprt_load_transport() needs to support the netid "rdma6"
SUNRPC: Close a race with transport setup and module put
SUNRPC: Add a helper to return the transport identifier given a netid
NFS: Switch mount code to use xprt_find_transport_ident()
SUNRPC: Remove unused function xprt_load_transport() NFSv4/pNFS: Use connections to a DS that are all of the same protocol
family
pNFS: Add helpers for allocation/free of struct nfs4_pnfs_ds_addr NFSv4/pNFS: Store the transport type in struct nfs4_pnfs_ds_addr pNFS/flexfiles: Fix up layoutstats reporting for non-TCP transports
SUNRPC: Fix up open coded kmemdup_nul()
pNFS: Clean up open coded xdr string decoding

 fs/nfs/flexfilelayout/flexfilelayout.c |   9 +-
 fs/nfs/fs_context.c                    |  21 +++--
 fs/nfs/pnfs.h                          |   2 +
 fs/nfs/pnfs_nfs.c                      | 103 ++++++++++------------
 include/linux/sunrpc/xprt.h            |   3 +-
 net/sunrpc/xdr.c                       |   4 +-
 net/sunrpc/xprt.c                      | 117 ++++++++++++++++++-------
 net/sunrpc/xprtrdma/module.c           |   1 +
 net/sunrpc/xprtrdma/transport.c        |   1 +
 net/sunrpc/xprtsock.c                  |   4 +
 10 files changed, 159 insertions(+), 106 deletions(-)
Signature · 1 line
2.28.0
author maintainer
#1 lore raw

PATCH v3 1/11 SUNRPC: xprt_load_transport() needs to support the netid "rdma6"

From: Trond Myklebust <[email protected]>

According to RFC5666, the correct netid for an IPv6 addressed RDMA transport is "rdma6", which we've supported as a mount option since Linux-4.7. The problem is when we try to load the module "xprtrdma6", that will fail, since there is no modulealias of that name.

Fixes 181342c5ebe8 ("xprtrdma: Add rdma6 option to support NFS/RDMA IPv6")
---
 include/linux/sunrpc/xprt.h     |  1 +
 net/sunrpc/xprt.c               | 65 +++++++++++++++++++++++++--------
 net/sunrpc/xprtrdma/module.c    |  1 +
 net/sunrpc/xprtrdma/transport.c |  1 +
 net/sunrpc/xprtsock.c           |  4 ++
 5 files changed, 56 insertions(+), 16 deletions(-)
include/linux/sunrpc/xprt.h +1 -0
index a603d48d2b2c..3ac5037d1c3d 100644
@@ -330,6 +330,7 @@ struct xprt_class {
 	struct rpc_xprt *	(*setup)(struct xprt_create *);
 	struct module		*owner;
 	char			name[32];
+	const char *		netid[];
 };
 
 /*
net/sunrpc/xprt.c +49 -16

80 lines not on this page · raw message

net/sunrpc/xprtrdma/module.c +1 -0
index 620327c01302..45c5b41ac8dc 100644
@@ -24,6 +24,7 @@ MODULE_DESCRIPTION("RPC/RDMA Transport");
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_ALIAS("svcrdma");
 MODULE_ALIAS("xprtrdma");
+MODULE_ALIAS("rpcrdma6");
 
 static void __exit rpc_rdma_cleanup(void)
 {
net/sunrpc/xprtrdma/transport.c +1 -0
index 8915e42240d3..035060c05fd5 100644
@@ -768,6 +768,7 @@ static struct xprt_class xprt_rdma = {
 	.owner			= THIS_MODULE,
 	.ident			= XPRT_TRANSPORT_RDMA,
 	.setup			= xprt_setup_rdma,
+	.netid			= { "rdma", "rdma6", "" },
 };
 
 void xprt_rdma_cleanup(void)
net/sunrpc/xprtsock.c +4 -0
index 7090bbee0ec5..c93ff70da3f9 100644
@@ -3059,6 +3059,7 @@ static struct xprt_class	xs_local_transport = {
 	.owner		= THIS_MODULE,
 	.ident		= XPRT_TRANSPORT_LOCAL,
 	.setup		= xs_setup_local,
+	.netid		= { "" },
 };
 
 static struct xprt_class	xs_udp_transport = {
@@ -3067,6 +3068,7 @@ static struct xprt_class	xs_udp_transport = {
 	.owner		= THIS_MODULE,
 	.ident		= XPRT_TRANSPORT_UDP,
 	.setup		= xs_setup_udp,
+	.netid		= { "udp", "udp6", "" },
 };
 
 static struct xprt_class	xs_tcp_transport = {
@@ -3075,6 +3077,7 @@ static struct xprt_class	xs_tcp_transport = {
 	.owner		= THIS_MODULE,
 	.ident		= XPRT_TRANSPORT_TCP,
 	.setup		= xs_setup_tcp,
+	.netid		= { "tcp", "tcp6", "" },
 };
 
 static struct xprt_class	xs_bc_tcp_transport = {
@@ -3083,6 +3086,7 @@ static struct xprt_class	xs_bc_tcp_transport = {
 	.owner		= THIS_MODULE,
 	.ident		= XPRT_TRANSPORT_BC_TCP,
 	.setup		= xs_setup_bc_tcp,
+	.netid		= { "" },
 };
 
 /**
Signature · 1 line
2.28.0
author maintainer
#2 lore raw

PATCH v3 2/11 SUNRPC: Close a race with transport setup and module put

From: Trond Myklebust <[email protected]>

After we've looked up the transport module, we need to ensure it can't go away until we've finished running the transport setup code.

---
 net/sunrpc/xprt.c | 44 +++++++++++++++++++++++++++++++++-----------
 1 file changed, 33 insertions(+), 11 deletions(-)
net/sunrpc/xprt.c +33 -11
index 1660a4b03352..29de33ea53d6 100644
@@ -157,6 +157,32 @@ xprt_class_release(const struct xprt_class *t)
 	module_put(t->owner);
 }
 
+static const struct xprt_class *
+xprt_class_find_by_ident_locked(int ident)
+{
+	const struct xprt_class *t;
+
+	list_for_each_entry(t, &xprt_list, list) {
+		if (t->ident != ident)
+			continue;
+		if (!try_module_get(t->owner))
+			continue;
+		return t;
+	}
+	return NULL;
+}
+
+static const struct xprt_class *
+xprt_class_find_by_ident(int ident)
+{
+	const struct xprt_class *t;
+
+	spin_lock(&xprt_list_lock);
+	t = xprt_class_find_by_ident_locked(ident);
+	spin_unlock(&xprt_list_lock);
+	return t;
+}
+
 static const struct xprt_class *
 xprt_class_find_by_netid_locked(const char *netid)
 {
@@ -1929,21 +1955,17 @@ static void xprt_init(struct rpc_xprt *xprt, struct net *net)
 struct rpc_xprt *xprt_create_transport(struct xprt_create *args)
 {
 	struct rpc_xprt	*xprt;
-	struct xprt_class *t;
+	const struct xprt_class *t;
 
-	spin_lock(&xprt_list_lock);
-	list_for_each_entry(t, &xprt_list, list) {
-		if (t->ident == args->ident) {
-			spin_unlock(&xprt_list_lock);
-			goto found;
-		}
+	t = xprt_class_find_by_ident(args->ident);
+	if (!t) {
+		dprintk("RPC: transport (%d) not supported\n", args->ident);
+		return ERR_PTR(-EIO);
 	}
-	spin_unlock(&xprt_list_lock);
-	dprintk("RPC: transport (%d) not supported\n", args->ident);
-	return ERR_PTR(-EIO);
 
-found:
 	xprt = t->setup(args);
+	xprt_class_release(t);
+
 	if (IS_ERR(xprt))
 		goto out;
 	if (args->flags & XPRT_CREATE_NO_IDLE_TIMEOUT)
Signature · 1 line
2.28.0
author maintainer
#3 lore raw

PATCH v3 3/11 SUNRPC: Add a helper to return the transport identifier given a netid

From: Trond Myklebust <[email protected]>

---
 include/linux/sunrpc/xprt.h |  1 +
 net/sunrpc/xprt.c           | 25 +++++++++++++++++++++----
 2 files changed, 22 insertions(+), 4 deletions(-)
include/linux/sunrpc/xprt.h +1 -0
index 3ac5037d1c3d..f7b75c72f80e 100644
@@ -386,6 +386,7 @@ xprt_disable_swap(struct rpc_xprt *xprt)
 int			xprt_register_transport(struct xprt_class *type);
 int			xprt_unregister_transport(struct xprt_class *type);
 int			xprt_load_transport(const char *);
+int			xprt_find_transport_ident(const char *);
 void			xprt_wait_for_reply_request_def(struct rpc_task *task);
 void			xprt_wait_for_reply_request_rtt(struct rpc_task *task);
 void			xprt_wake_pending_tasks(struct rpc_xprt *xprt, int status);
net/sunrpc/xprt.c +21 -4
index 29de33ea53d6..1016265d5e53 100644
@@ -219,22 +219,39 @@ xprt_class_find_by_netid(const char *netid)
 }
 
 /**
- * xprt_load_transport - load a transport implementation
+ * xprt_find_transport_ident - convert a netid into a transport identifier
  * @netid: transport to load
  *
  * Returns:
- * 0:		transport successfully loaded
+ * > 0:		transport identifier
  * -ENOENT:	transport module not available
  */
-int xprt_load_transport(const char *netid)
+int xprt_find_transport_ident(const char *netid)
 {
 	const struct xprt_class *t;
+	int ret;
 
 	t = xprt_class_find_by_netid(netid);
 	if (!t)
 		return -ENOENT;
+	ret = t->ident;
 	xprt_class_release(t);
-	return 0;
+	return ret;
+}
+EXPORT_SYMBOL_GPL(xprt_find_transport_ident);
+
+/**
+ * xprt_load_transport - load a transport implementation
+ * @netid: transport to load
+ *
+ * Returns:
+ * 0:		transport successfully loaded
+ * -ENOENT:	transport module not available
+ */
+int xprt_load_transport(const char *netid)
+{
+	int ret = xprt_find_transport_ident(netid);
+	return ret < 0 ? ret : 0;
 }
 EXPORT_SYMBOL_GPL(xprt_load_transport);
 
Signature · 1 line
2.28.0
author maintainer
#4 lore raw

PATCH v3 4/11 NFS: Switch mount code to use xprt_find_transport_ident()

From: Trond Myklebust <[email protected]>

Switch the mount code to use xprt_find_transport_ident() and to check the results before allowing the mount to proceed.

---
 fs/nfs/fs_context.c | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)
fs/nfs/fs_context.c +12 -9
index 29ec8b09a52d..06894bcdea2d 100644
@@ -510,13 +510,12 @@ static int nfs_fs_context_parse_param(struct fs_context *fc,
 		ctx->nfs_server.protocol = XPRT_TRANSPORT_UDP;
 		break;
 	case Opt_tcp:
-		ctx->flags |= NFS_MOUNT_TCP;
-		ctx->nfs_server.protocol = XPRT_TRANSPORT_TCP;
-		break;
 	case Opt_rdma:
 		ctx->flags |= NFS_MOUNT_TCP; /* for side protocols */
-		ctx->nfs_server.protocol = XPRT_TRANSPORT_RDMA;
-		xprt_load_transport(param->key);
+		ret = xprt_find_transport_ident(param->key);
+		if (ret < 0)
+			goto out_bad_transport;
+		ctx->nfs_server.protocol = ret;
 		break;
 	case Opt_acl:
 		if (result.negated)
@@ -670,11 +669,13 @@ static int nfs_fs_context_parse_param(struct fs_context *fc,
 		case Opt_xprt_rdma:
 			/* vector side protocols to TCP */
 			ctx->flags |= NFS_MOUNT_TCP;
-			ctx->nfs_server.protocol = XPRT_TRANSPORT_RDMA;
-			xprt_load_transport(param->string);
+			ret = xprt_find_transport_ident(param->string);
+			if (ret < 0)
+				goto out_bad_transport;
+			ctx->nfs_server.protocol = ret;
 			break;
 		default:
-			return nfs_invalf(fc, "NFS: Unrecognized transport protocol");
+			goto out_bad_transport;
 		}
 
 		ctx->protofamily = protofamily;
@@ -697,7 +698,7 @@ static int nfs_fs_context_parse_param(struct fs_context *fc,
 			break;
 		case Opt_xprt_rdma: /* not used for side protocols */
 		default:
-			return nfs_invalf(fc, "NFS: Unrecognized transport protocol");
+			goto out_bad_transport;
 		}
 		ctx->mountfamily = mountfamily;
 		break;
@@ -787,6 +788,8 @@ static int nfs_fs_context_parse_param(struct fs_context *fc,
 	return nfs_invalf(fc, "NFS: Bad IP address specified");
 out_of_bounds:
 	return nfs_invalf(fc, "NFS: Value for '%s' out of range", param->key);
+out_bad_transport:
+	return nfs_invalf(fc, "NFS: Unrecognized transport protocol");
 }
 
 /*
Signature · 1 line
2.28.0
author maintainer
#5 lore raw

PATCH v3 5/11 SUNRPC: Remove unused function xprt_load_transport()

From: Trond Myklebust <[email protected]>

---
 include/linux/sunrpc/xprt.h |  1 -
 net/sunrpc/xprt.c           | 15 ---------------
 2 files changed, 16 deletions(-)
include/linux/sunrpc/xprt.h +0 -1
index f7b75c72f80e..d2e97ee802af 100644
@@ -385,7 +385,6 @@ xprt_disable_swap(struct rpc_xprt *xprt)
  */
 int			xprt_register_transport(struct xprt_class *type);
 int			xprt_unregister_transport(struct xprt_class *type);
-int			xprt_load_transport(const char *);
 int			xprt_find_transport_ident(const char *);
 void			xprt_wait_for_reply_request_def(struct rpc_task *task);
 void			xprt_wait_for_reply_request_rtt(struct rpc_task *task);
net/sunrpc/xprt.c +0 -15
index 1016265d5e53..9018d73575d6 100644
@@ -240,21 +240,6 @@ int xprt_find_transport_ident(const char *netid)
 }
 EXPORT_SYMBOL_GPL(xprt_find_transport_ident);
 
-/**
- * xprt_load_transport - load a transport implementation
- * @netid: transport to load
- *
- * Returns:
- * 0:		transport successfully loaded
- * -ENOENT:	transport module not available
- */
-int xprt_load_transport(const char *netid)
-{
-	int ret = xprt_find_transport_ident(netid);
-	return ret < 0 ? ret : 0;
-}
-EXPORT_SYMBOL_GPL(xprt_load_transport);
-
 static void xprt_clear_locked(struct rpc_xprt *xprt)
 {
 	xprt->snd_task = NULL;
Signature · 1 line
2.28.0
author maintainer
#6 lore raw

PATCH v3 6/11 NFSv4/pNFS: Use connections to a DS that are all of the same protocol family

From: Trond Myklebust <[email protected]>

If the pNFS metadata server advertises multiple addresses for the same data server, we should try to connect to just one protocol family and transport type on the assumption that homogeneity will improve performance.

---
 fs/nfs/pnfs_nfs.c | 5 +++++
 1 file changed, 5 insertions(+)
fs/nfs/pnfs_nfs.c +5 -0
index 679767ac258d..7027dac41cc7 100644
@@ -860,6 +860,9 @@ static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv,
 				.addrlen = da->da_addrlen,
 				.servername = clp->cl_hostname,
 			};
+
+			if (da->da_addr.ss_family != clp->cl_addr.ss_family)
+				continue;
 			/* Add this address as an alias */
 			rpc_clnt_add_xprt(clp->cl_rpcclient, &xprt_args,
 					rpc_clnt_test_and_add_xprt, NULL);
@@ -920,6 +923,8 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv,
 				.data = &xprtdata,
 			};
 
+			if (da->da_addr.ss_family != clp->cl_addr.ss_family)
+				continue;
 			/**
 			* Test this address for session trunking and
 			* add as an alias
Signature · 1 line
2.28.0
author maintainer
#7 lore raw

PATCH v3 7/11 pNFS: Add helpers for allocation/free of struct nfs4_pnfs_ds_addr

From: Trond Myklebust <[email protected]>

---
 fs/nfs/pnfs_nfs.c | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)
fs/nfs/pnfs_nfs.c +16 -5
index 7027dac41cc7..8c5921be41f8 100644
@@ -661,6 +661,20 @@ _data_server_lookup_locked(const struct list_head *dsaddrs)
 	return NULL;
 }
 
+static struct nfs4_pnfs_ds_addr *nfs4_pnfs_ds_addr_alloc(gfp_t gfp_flags)
+{
+	struct nfs4_pnfs_ds_addr *da = kzalloc(sizeof(*da), gfp_flags);
+	if (da)
+		INIT_LIST_HEAD(&da->da_node);
+	return da;
+}
+
+static void nfs4_pnfs_ds_addr_free(struct nfs4_pnfs_ds_addr *da)
+{
+	kfree(da->da_remotestr);
+	kfree(da);
+}
+
 static void destroy_ds(struct nfs4_pnfs_ds *ds)
 {
 	struct nfs4_pnfs_ds_addr *da;
@@ -676,8 +690,7 @@ static void destroy_ds(struct nfs4_pnfs_ds *ds)
 				      struct nfs4_pnfs_ds_addr,
 				      da_node);
 		list_del_init(&da->da_node);
-		kfree(da->da_remotestr);
-		kfree(da);
+		nfs4_pnfs_ds_addr_free(da);
 	}
 
 	kfree(ds->ds_remotestr);
@@ -1094,12 +1107,10 @@ nfs4_decode_mp_ds_addr(struct net *net, struct xdr_stream *xdr, gfp_t gfp_flags)
 	}
 	*portstr = '\0';
 
-	da = kzalloc(sizeof(*da), gfp_flags);
+	da = nfs4_pnfs_ds_addr_alloc(gfp_flags);
 	if (unlikely(!da))
 		goto out_free_buf;
 
-	INIT_LIST_HEAD(&da->da_node);
-
 	if (!rpc_pton(net, buf, portstr-buf, (struct sockaddr *)&da->da_addr,
 		      sizeof(da->da_addr))) {
 		dprintk("%s: error parsing address %s\n", __func__, buf);
Signature · 1 line
2.28.0
author maintainer
#8 lore raw

PATCH v3 8/11 NFSv4/pNFS: Store the transport type in struct nfs4_pnfs_ds_addr

From: Trond Myklebust <[email protected]>

We want to enable RDMA and UDP as valid transport methods if a GETDEVICEINFO call specifies it. Do so by adding a parser for the netid that translates it to an appropriate argument for the RPC transport layer.

---
 fs/nfs/pnfs.h     |  2 ++
 fs/nfs/pnfs_nfs.c | 34 +++++++++++++++++++---------------
 2 files changed, 21 insertions(+), 15 deletions(-)
fs/nfs/pnfs.h +2 -0
index 2661c44c62db..f618c49697bb 100644
@@ -51,6 +51,8 @@ struct nfs4_pnfs_ds_addr {
 	size_t			da_addrlen;
 	struct list_head	da_node;  /* nfs4_pnfs_dev_hlist dev_dslist */
 	char			*da_remotestr;	/* human readable addr+port */
+	const char		*da_netid;
+	int			da_transport;
 };
 
 struct nfs4_pnfs_ds {
fs/nfs/pnfs_nfs.c +19 -15
index 8c5921be41f8..b96ecb395310 100644
@@ -672,6 +672,7 @@ static struct nfs4_pnfs_ds_addr *nfs4_pnfs_ds_addr_alloc(gfp_t gfp_flags)
 static void nfs4_pnfs_ds_addr_free(struct nfs4_pnfs_ds_addr *da)
 {
 	kfree(da->da_remotestr);
+	kfree(da->da_netid);
 	kfree(da);
 }
 
@@ -867,13 +868,15 @@ static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv,
 
 		if (!IS_ERR(clp)) {
 			struct xprt_create xprt_args = {
-				.ident = XPRT_TRANSPORT_TCP,
+				.ident = da->da_transport,
 				.net = clp->cl_net,
 				.dstaddr = (struct sockaddr *)&da->da_addr,
 				.addrlen = da->da_addrlen,
 				.servername = clp->cl_hostname,
 			};
 
+			if (da->da_transport != clp->cl_proto)
+				continue;
 			if (da->da_addr.ss_family != clp->cl_addr.ss_family)
 				continue;
 			/* Add this address as an alias */
@@ -883,7 +886,7 @@ static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv,
 		}
 		clp = get_v3_ds_connect(mds_srv,
 				(struct sockaddr *)&da->da_addr,
-				da->da_addrlen, IPPROTO_TCP,
+				da->da_addrlen, da->da_transport,
 				timeo, retrans);
 		if (IS_ERR(clp))
 			continue;
@@ -921,7 +924,7 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv,
 
 		if (!IS_ERR(clp) && clp->cl_mvops->session_trunk) {
 			struct xprt_create xprt_args = {
-				.ident = XPRT_TRANSPORT_TCP,
+				.ident = da->da_transport,
 				.net = clp->cl_net,
 				.dstaddr = (struct sockaddr *)&da->da_addr,
 				.addrlen = da->da_addrlen,
@@ -936,6 +939,8 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv,
 				.data = &xprtdata,
 			};
 
+			if (da->da_transport != clp->cl_proto)
+				continue;
 			if (da->da_addr.ss_family != clp->cl_addr.ss_family)
 				continue;
 			/**
@@ -950,8 +955,9 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv,
 		} else {
 			clp = nfs4_set_ds_client(mds_srv,
 						(struct sockaddr *)&da->da_addr,
-						da->da_addrlen, IPPROTO_TCP,
-						timeo, retrans, minor_version);
+						da->da_addrlen,
+						da->da_transport, timeo,
+						retrans, minor_version);
 			if (IS_ERR(clp))
 				continue;
 
@@ -1042,8 +1048,8 @@ nfs4_decode_mp_ds_addr(struct net *net, struct xdr_stream *xdr, gfp_t gfp_flags)
 	int nlen, rlen;
 	int tmp[2];
 	__be32 *p;
-	char *netid, *match_netid;
-	size_t len, match_netid_len;
+	char *netid;
+	size_t len;
 	char *startsep = "";
 	char *endsep = "";
 
@@ -1125,15 +1131,11 @@ nfs4_decode_mp_ds_addr(struct net *net, struct xdr_stream *xdr, gfp_t gfp_flags)
 	case AF_INET:
 		((struct sockaddr_in *)&da->da_addr)->sin_port = port;
 		da->da_addrlen = sizeof(struct sockaddr_in);
-		match_netid = "tcp";
-		match_netid_len = 3;
 		break;
 
 	case AF_INET6:
 		((struct sockaddr_in6 *)&da->da_addr)->sin6_port = port;
 		da->da_addrlen = sizeof(struct sockaddr_in6);
-		match_netid = "tcp6";
-		match_netid_len = 4;
 		startsep = "[";
 		endsep = "]";
 		break;
@@ -1144,12 +1146,15 @@ nfs4_decode_mp_ds_addr(struct net *net, struct xdr_stream *xdr, gfp_t gfp_flags)
 		goto out_free_da;
 	}
 
-	if (nlen != match_netid_len || strncmp(netid, match_netid, nlen)) {
-		dprintk("%s: ERROR: r_netid \"%s\" != \"%s\"\n",
-			__func__, netid, match_netid);
+	da->da_transport = xprt_find_transport_ident(netid);
+	if (da->da_transport < 0) {
+		dprintk("%s: ERROR: unknown r_netid \"%s\"\n",
+			__func__, netid);
 		goto out_free_da;
 	}
 
+	da->da_netid = netid;
+
 	/* save human readable address */
 	len = strlen(startsep) + strlen(buf) + strlen(endsep) + 7;
 	da->da_remotestr = kzalloc(len, gfp_flags);
@@ -1161,7 +1166,6 @@ nfs4_decode_mp_ds_addr(struct net *net, struct xdr_stream *xdr, gfp_t gfp_flags)
 
 	dprintk("%s: Parsed DS addr %s\n", __func__, da->da_remotestr);
 	kfree(buf);
-	kfree(netid);
 	return da;
 
 out_free_da:
Signature · 1 line
2.28.0
author maintainer
#9 lore raw

PATCH v3 9/11 pNFS/flexfiles: Fix up layoutstats reporting for non-TCP transports

From: Trond Myklebust <[email protected]>

Ensure that we report the correct netid when using UDP or RDMA transports to the DSes.

---
 fs/nfs/flexfilelayout/flexfilelayout.c | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)
fs/nfs/flexfilelayout/flexfilelayout.c +2 -7
index a163533446fa..59ae36bf5cc0 100644
@@ -2269,7 +2269,6 @@ ff_layout_encode_netaddr(struct xdr_stream *xdr, struct nfs4_pnfs_ds_addr *da)
 	struct sockaddr *sap = (struct sockaddr *)&da->da_addr;
 	char portbuf[RPCBIND_MAXUADDRPLEN];
 	char addrbuf[RPCBIND_MAXUADDRLEN];
-	char *netid;
 	unsigned short port;
 	int len, netid_len;
 	__be32 *p;
@@ -2279,18 +2278,13 @@ ff_layout_encode_netaddr(struct xdr_stream *xdr, struct nfs4_pnfs_ds_addr *da)
 		if (ff_layout_ntop4(sap, addrbuf, sizeof(addrbuf)) == 0)
 			return;
 		port = ntohs(((struct sockaddr_in *)sap)->sin_port);
-		netid = "tcp";
-		netid_len = 3;
 		break;
 	case AF_INET6:
 		if (ff_layout_ntop6_noscopeid(sap, addrbuf, sizeof(addrbuf)) == 0)
 			return;
 		port = ntohs(((struct sockaddr_in6 *)sap)->sin6_port);
-		netid = "tcp6";
-		netid_len = 4;
 		break;
 	default:
-		/* we only support tcp and tcp6 */
 		WARN_ON_ONCE(1);
 		return;
 	}
@@ -2298,8 +2292,9 @@ ff_layout_encode_netaddr(struct xdr_stream *xdr, struct nfs4_pnfs_ds_addr *da)
 	snprintf(portbuf, sizeof(portbuf), ".%u.%u", port >> 8, port & 0xff);
 	len = strlcat(addrbuf, portbuf, sizeof(addrbuf));
 
+	netid_len = strlen(da->da_netid);
 	p = xdr_reserve_space(xdr, 4 + netid_len);
-	xdr_encode_opaque(p, netid, netid_len);
+	xdr_encode_opaque(p, da->da_netid, netid_len);
 
 	p = xdr_reserve_space(xdr, 4 + len);
 	xdr_encode_opaque(p, addrbuf, len);
Signature · 1 line
2.28.0
author maintainer
#10 lore raw

PATCH v3 10/11 SUNRPC: Fix up open coded kmemdup_nul()

From: Trond Myklebust <[email protected]>

---
 net/sunrpc/xdr.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
net/sunrpc/xdr.c +1 -3
index 71e03b930b70..b1cda6d85ded 100644
@@ -1942,10 +1942,8 @@ ssize_t xdr_stream_decode_string_dup(struct xdr_stream *xdr, char **str,
 
 	ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen);
 	if (ret > 0) {
-		char *s = kmalloc(ret + 1, gfp_flags);
+		char *s = kmemdup_nul(p, ret, gfp_flags);
 		if (s != NULL) {
-			memcpy(s, p, ret);
-			s[ret] = '\0';
 			*str = s;
 			return strlen(s);
 		}
Signature · 1 line
2.28.0
author maintainer
#11 lore raw

PATCH v3 11/11 pNFS: Clean up open coded xdr string decoding

From: Trond Myklebust <[email protected]>

Use the existing xdr_stream_decode_string_dup() to safely decode into kmalloced strings.

---
 fs/nfs/pnfs_nfs.c | 43 +++++++------------------------------------
 1 file changed, 7 insertions(+), 36 deletions(-)
fs/nfs/pnfs_nfs.c +7 -36
index b96ecb395310..1babf56faf66 100644
@@ -1045,9 +1045,8 @@ nfs4_decode_mp_ds_addr(struct net *net, struct xdr_stream *xdr, gfp_t gfp_flags)
 	struct nfs4_pnfs_ds_addr *da = NULL;
 	char *buf, *portstr;
 	__be16 port;
-	int nlen, rlen;
+	ssize_t nlen, rlen;
 	int tmp[2];
-	__be32 *p;
 	char *netid;
 	size_t len;
 	char *startsep = "";
@@ -1055,45 +1054,17 @@ nfs4_decode_mp_ds_addr(struct net *net, struct xdr_stream *xdr, gfp_t gfp_flags)
 
 
 	/* r_netid */
-	p = xdr_inline_decode(xdr, 4);
-	if (unlikely(!p))
+	nlen = xdr_stream_decode_string_dup(xdr, &netid, XDR_MAX_NETOBJ,
+					    gfp_flags);
+	if (unlikely(nlen < 0))
 		goto out_err;
-	nlen = be32_to_cpup(p++);
-
-	p = xdr_inline_decode(xdr, nlen);
-	if (unlikely(!p))
-		goto out_err;
-
-	netid = kmalloc(nlen+1, gfp_flags);
-	if (unlikely(!netid))
-		goto out_err;
-
-	netid[nlen] = '\0';
-	memcpy(netid, p, nlen);
 
 	/* r_addr: ip/ip6addr with port in dec octets - see RFC 5665 */
-	p = xdr_inline_decode(xdr, 4);
-	if (unlikely(!p))
-		goto out_free_netid;
-	rlen = be32_to_cpup(p);
-
-	p = xdr_inline_decode(xdr, rlen);
-	if (unlikely(!p))
-		goto out_free_netid;
-
 	/* port is ".ABC.DEF", 8 chars max */
-	if (rlen > INET6_ADDRSTRLEN + IPV6_SCOPE_ID_LEN + 8) {
-		dprintk("%s: Invalid address, length %d\n", __func__,
-			rlen);
+	rlen = xdr_stream_decode_string_dup(xdr, &buf, INET6_ADDRSTRLEN +
+					    IPV6_SCOPE_ID_LEN + 8, gfp_flags);
+	if (unlikely(rlen < 0))
 		goto out_free_netid;
-	}
-	buf = kmalloc(rlen + 1, gfp_flags);
-	if (!buf) {
-		dprintk("%s: Not enough memory\n", __func__);
-		goto out_free_netid;
-	}
-	buf[rlen] = '\0';
-	memcpy(buf, p, rlen);
 
 	/* replace port '.' with '-' */
 	portstr = strrchr(buf, '.');
Signature · 1 line
2.28.0
#12 lore raw
In reply to [email protected]
Quoted 11 lines · On Tue, 2020-11-10 at 18:18 -0500, [email protected] wrote:

From: Trond Myklebust <[email protected]>

Add support for connecting to the pNFS files/flexfiles data servers through RDMA, assuming that the GETDEVICEINFO call advertises that support.

v2: Fix layoutstats encoding for pNFS/flexfiles.
v3: Move most of the netid handling into the SUNRPC and RDMA modules. Fix up the mount code to benefit more from automated loading of SUNRPC transport modules.

Note that one cleanup that I did not perform, but which really could be useful should we want to add more transport mechanisms, is to move the code to parse stringified addresses (in particular IETF style "universal addresses") into the transport modules so that the actual parsing of mount and pNFS transport info can be automatically extended when new transport modules are added.

Signature · 3 lines
Trond Myklebust
Linux NFS client maintainer, Hammerspace
[email protected]
#13 lore raw

Hi Trond,

whit this changes (3ee69a14f92d74ced2647140b3799511ba4f3fa5) I see an infinite loop of LAYOUTGET->GETDEVICEINFO->LAYOUTRETURN without any attempt to connect to a DS.

This is how the response to LAYOUTGET looks like.

Network File System, Ops(2): SEQUENCE GETDEVINFO
[Program Version: 4]
[V4 Procedure: COMPOUND (1)]
Status: NFS4_OK (0)
Tag: <EMPTY>
length: 0
contents: <EMPTY>
Operations (count: 2)
Opcode: SEQUENCE (53)
Opcode: GETDEVINFO (47)
Status: NFS4_OK (0)
layout type: LAYOUT4_FLEX_FILES (4)
r_netid: tcp
length: 3
contents: tcp
fill bytes: opaque data
r_addr: 131.169.191.143.125.49
length: 22
contents: 131.169.191.143.125.49 fill bytes: opaque data
version: 4
minorversion: 1
max_rsize: 1048576
max_wsize: 1048576
tightly coupled: Yes
notify_mask: 0x00000006 (Change, Delete)
notify_type: Change (1)
notify_type: Delete (2)
[Main Opcode: GETDEVINFO (47)]

The MDS is mounted with IPv4. I can provide the full packet trace, if needed.

Regards,
Tigran.

----- Original Message -----

Quoted 29 lines

From: "trondmy" <[email protected]>
To: "linux-nfs" <[email protected]>
Sent: Wednesday, 11 November, 2020 00:42:31
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Quoted 11 lines · On Tue, 2020-11-10 at 18:18 -0500, [email protected] wrote:

From: Trond Myklebust <[email protected]>

Add support for connecting to the pNFS files/flexfiles data servers through RDMA, assuming that the GETDEVICEINFO call advertises that support.

v2: Fix layoutstats encoding for pNFS/flexfiles.
v3: Move most of the netid handling into the SUNRPC and RDMA modules. Fix up the mount code to benefit more from automated loading of SUNRPC transport modules.

Note that one cleanup that I did not perform, but which really could be useful should we want to add more transport mechanisms, is to move the code to parse stringified addresses (in particular IETF style "universal addresses") into the transport modules so that the actual parsing of mount and pNFS transport info can be automatically extended when new transport modules are added.

--
Trond Myklebust
Linux NFS client maintainer, Hammerspace
[email protected]

#14 lore raw

After more testing, it looks like that client doesn't like notification bitmap:

[31576.789492] --> _nfs4_proc_getdeviceinfo
[31576.789503] --> nfs41_call_sync_prepare data->seq_server 000000001d17c43e
[31576.789507] --> nfs4_alloc_slot used_slots=0000 highest_used=4294967295 max_slots=16
[31576.789510] <-- nfs4_alloc_slot used_slots=0001 highest_used=0 slotid=0
[31576.789527] encode_sequence: sessionid=2910695007:150995712:0:16777216 seqid=92462 slotid=0 max_slotid=0 cache_this=0
[31576.789991] decode_getdeviceinfo: unsupported notification
[31576.790003] --> nfs4_alloc_slot used_slots=0001 highest_used=0 max_slots=16
[31576.790007] <-- nfs4_alloc_slot used_slots=0003 highest_used=1 slotid=1
[31576.790010] nfs4_free_slot: slotid 1 highest_used_slotid 0
[31576.790013] nfs41_sequence_process: Error 0 free the slot
[31576.790017] nfs4_free_slot: slotid 0 highest_used_slotid 4294967295
[31576.790030] <-- _nfs4_proc_getdeviceinfo status=-5
[31576.790034] nfs4_get_device_info getdevice info returns -5
[31576.790084] <-- nfs4_get_device_info d 0000000000000000

Tigran.

----- Original Message -----

Quoted 85 lines

From: "Tigran Mkrtchyan" <[email protected]>
To: "trondmy" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Friday, 13 November, 2020 13:48:07
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Hi Trond,

whit this changes (3ee69a14f92d74ced2647140b3799511ba4f3fa5) I see an infinite loop
of LAYOUTGET->GETDEVICEINFO->LAYOUTRETURN without any attempt to connect to a DS.

This is how the response to LAYOUTGET looks like.

Network File System, Ops(2): SEQUENCE GETDEVINFO
[Program Version: 4]
[V4 Procedure: COMPOUND (1)]
Status: NFS4_OK (0)
Tag: <EMPTY>
length: 0
contents: <EMPTY>
Operations (count: 2)
Opcode: SEQUENCE (53)
Opcode: GETDEVINFO (47)
Status: NFS4_OK (0)
layout type: LAYOUT4_FLEX_FILES (4)
r_netid: tcp
length: 3
contents: tcp
fill bytes: opaque data
r_addr: 131.169.191.143.125.49
length: 22
contents: 131.169.191.143.125.49 fill bytes: opaque data
version: 4
minorversion: 1
max_rsize: 1048576
max_wsize: 1048576
tightly coupled: Yes
notify_mask: 0x00000006 (Change, Delete)
notify_type: Change (1)
notify_type: Delete (2)
[Main Opcode: GETDEVINFO (47)]

The MDS is mounted with IPv4. I can provide the full packet trace, if needed.

Regards,
Tigran.

----- Original Message -----

Quoted 30 lines

From: "trondmy" <[email protected]>
To: "linux-nfs" <[email protected]>
Sent: Wednesday, 11 November, 2020 00:42:31
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Quoted 11 lines · On Tue, 2020-11-10 at 18:18 -0500, [email protected] wrote:

From: Trond Myklebust <[email protected]>

Add support for connecting to the pNFS files/flexfiles data servers through RDMA, assuming that the GETDEVICEINFO call advertises that support.

v2: Fix layoutstats encoding for pNFS/flexfiles.
v3: Move most of the netid handling into the SUNRPC and RDMA modules. Fix up the mount code to benefit more from automated loading of SUNRPC transport modules.

Note that one cleanup that I did not perform, but which really could be useful should we want to add more transport mechanisms, is to move the code to parse stringified addresses (in particular IETF style "universal addresses") into the transport modules so that the actual parsing of mount and pNFS transport info can be automatically extended when new transport modules are added.

--
Trond Myklebust
Linux NFS client maintainer, Hammerspace
[email protected]

#15 lore raw
Quoted 5 lines · Tigran

After more testing, it looks like that client doesn't like notification bitmap:

Quoted 11 lines
[31576.789492] --> _nfs4_proc_getdeviceinfo
[31576.789503] --> nfs41_call_sync_prepare data->seq_server
000000001d17c43e
[31576.789507] --> nfs4_alloc_slot used_slots=0000
highest_used=4294967295 max_slots=16
[31576.789510] <-- nfs4_alloc_slot used_slots=0001 highest_used=0
slotid=0
[31576.789527] encode_sequence:
sessionid=2910695007:150995712:0:16777216 seqid=92462 slotid=0
max_slotid=0 cache_this=0
[31576.789991] decode_getdeviceinfo: unsupported notification

According to this, you appear to be returning a deviceinfo bitmap with at least one non-zero entry that is not in the first 32-bit word. We only ask for notifications for NOTIFY_DEVICEID4_CHANGE and NOTIFY_DEVICEID4_DELETE, so we only expect bitmap[0] to have non-zero entries.

Quoted 115 lines
[31576.790003] --> nfs4_alloc_slot used_slots=0001 highest_used=0
max_slots=16
[31576.790007] <-- nfs4_alloc_slot used_slots=0003 highest_used=1
slotid=1
[31576.790010] nfs4_free_slot: slotid 1 highest_used_slotid 0
[31576.790013] nfs41_sequence_process: Error 0 free the slot
[31576.790017] nfs4_free_slot: slotid 0 highest_used_slotid
4294967295
[31576.790030] <-- _nfs4_proc_getdeviceinfo status=-5
[31576.790034] nfs4_get_device_info getdevice info returns -5
[31576.790084] <-- nfs4_get_device_info d 0000000000000000

Tigran.

----- Original Message -----

Quoted 98 lines

From: "Tigran Mkrtchyan" <[email protected]>
To: "trondmy" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Friday, 13 November, 2020 13:48:07
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Hi Trond,

whit this changes (3ee69a14f92d74ced2647140b3799511ba4f3fa5) I see an infinite
loop
of LAYOUTGET->GETDEVICEINFO->LAYOUTRETURN without any attempt to connect to a
DS.

This is how the response to LAYOUTGET looks like.

Network File System, Ops(2): SEQUENCE GETDEVINFO [Program Version: 4]
   [V4 Procedure: COMPOUND (1)]
   Status: NFS4_OK (0)
   Tag: <EMPTY>
       length: 0
       contents: <EMPTY>
   Operations (count: 2)
       Opcode: SEQUENCE (53)
       Opcode: GETDEVINFO (47)
           Status: NFS4_OK (0)
           layout type: LAYOUT4_FLEX_FILES (4)
           r_netid: tcp
               length: 3
               contents: tcp
               fill bytes: opaque data
           r_addr: 131.169.191.143.125.49
               length: 22
               contents: 131.169.191.143.125.49 fill bytes: opaque data
           version: 4
           minorversion: 1
           max_rsize: 1048576
           max_wsize: 1048576
           tightly coupled: Yes
           notify_mask: 0x00000006 (Change, Delete)
               notify_type: Change (1)
               notify_type: Delete (2) [Main Opcode: GETDEVINFO (47)]

The MDS is mounted with IPv4. I can provide the full packet trace, if needed.

Regards,
  Tigran.

----- Original Message -----

Quoted 39 lines

From: "trondmy" <[email protected]>
To: "linux-nfs" <[email protected]>
Sent: Wednesday, 11 November, 2020 00:42:31
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data
channels

Quoted 8 lines · On Tue, 2020-11-10 at 18:18 -0500, [email protected] wrote:

From: Trond Myklebust <[email protected]>

Add support for connecting to the pNFS files/flexfiles data servers
through RDMA, assuming that the GETDEVICEINFO call advertises that
support.

Quoted 7 lines

v2: Fix layoutstats encoding for pNFS/flexfiles.
v3: Move most of the netid handling into the SUNRPC and RDMA modules.
    Fix up the mount code to benefit more from automated loading of
    SUNRPC transport modules.

Note that one cleanup that I did not perform, but which really could be
useful should we want to add more transport mechanisms, is to move the
code to parse stringified addresses (in particular IETF style "universal addresses") into the transport modules so that the actual
parsing of mount and pNFS transport info can be automatically extended
when new transport modules are added.

--
Trond Myklebust
Linux NFS client maintainer, Hammerspace
[email protected]

Signature · 6 lines
Trond Myklebust
CTO, Hammerspace Inc
4984 El Camino Real, Suite 208
Los Altos, CA 94022
​
www.hammer.space
#16 lore raw

----- Original Message -----

Quoted 24 lines

From: "trondmy" <[email protected]>
To: "Tigran Mkrtchyan" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Friday, 13 November, 2020 23:45:00
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Quoted 5 lines · Tigran

After more testing, it looks like that client doesn't like notification bitmap:

Quoted 11 lines
[31576.789492] --> _nfs4_proc_getdeviceinfo
[31576.789503] --> nfs41_call_sync_prepare data->seq_server
000000001d17c43e
[31576.789507] --> nfs4_alloc_slot used_slots=0000
highest_used=4294967295 max_slots=16
[31576.789510] <-- nfs4_alloc_slot used_slots=0001 highest_used=0
slotid=0
[31576.789527] encode_sequence:
sessionid=2910695007:150995712:0:16777216 seqid=92462 slotid=0
max_slotid=0 cache_this=0
[31576.789991] decode_getdeviceinfo: unsupported notification
Quoted 5 lines

According to this, you appear to be returning a deviceinfo bitmap with at least one non-zero entry that is not in the first 32-bit word. We only ask for notifications for NOTIFY_DEVICEID4_CHANGE and NOTIFY_DEVICEID4_DELETE, so we only expect bitmap[0] to have non-zero entries.

according to packet capture only bitmap[0] has non zero bits set. This is the reply of compound starting from nfs staus code, tag length and so on.

0000   00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 35
0010   00 00 00 00 5f ae 7d ad 00 03 00 09 00 00 00 00
0020   00 00 00 01 00 00 00 4c 00 00 00 00 00 00 00 0f
0030   00 00 00 0f 00 00 00 00 00 00 00 2f 00 00 00 00
0040   00 00 00 04 00 00 00 40 00 00 00 01 00 00 00 03
0050   74 63 70 00 00 00 00 16 31 33 31 2e 31 36 39 2e
0060   31 39 31 2e 31 34 33 2e 31 32 35 2e 34 39 00 00
0070   00 00 00 01 00 00 00 04 00 00 00 01 00 10 00 00
0080   00 10 00 00 00 00 00 01 00 00 00 02 00 00 00 06
0090   00 00 00 00

the last 12 bytes : bitmap size, bitmap[0], bitmap[1]

This part of code in the didn't change since 2010, and I have no issues to use 5.8 kernel. I am pretty sure, that tests with 5.9 did pass as expected. I will try to bisec it.

Tigran.

Quoted 124 lines
Quoted 14 lines
[31576.790003] --> nfs4_alloc_slot used_slots=0001 highest_used=0
max_slots=16
[31576.790007] <-- nfs4_alloc_slot used_slots=0003 highest_used=1
slotid=1
[31576.790010] nfs4_free_slot: slotid 1 highest_used_slotid 0
[31576.790013] nfs41_sequence_process: Error 0 free the slot
[31576.790017] nfs4_free_slot: slotid 0 highest_used_slotid
4294967295
[31576.790030] <-- _nfs4_proc_getdeviceinfo status=-5
[31576.790034] nfs4_get_device_info getdevice info returns -5
[31576.790084] <-- nfs4_get_device_info d 0000000000000000
Quoted 102 lines

Tigran.

----- Original Message -----

Quoted 98 lines

From: "Tigran Mkrtchyan" <[email protected]>
To: "trondmy" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Friday, 13 November, 2020 13:48:07
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Hi Trond,

whit this changes (3ee69a14f92d74ced2647140b3799511ba4f3fa5) I see an infinite
loop
of LAYOUTGET->GETDEVICEINFO->LAYOUTRETURN without any attempt to connect to a
DS.

This is how the response to LAYOUTGET looks like.

Network File System, Ops(2): SEQUENCE GETDEVINFO [Program Version: 4]
   [V4 Procedure: COMPOUND (1)]
   Status: NFS4_OK (0)
   Tag: <EMPTY>
       length: 0
       contents: <EMPTY>
   Operations (count: 2)
       Opcode: SEQUENCE (53)
       Opcode: GETDEVINFO (47)
           Status: NFS4_OK (0)
           layout type: LAYOUT4_FLEX_FILES (4)
           r_netid: tcp
               length: 3
               contents: tcp
               fill bytes: opaque data
           r_addr: 131.169.191.143.125.49
               length: 22
               contents: 131.169.191.143.125.49 fill bytes: opaque data
           version: 4
           minorversion: 1
           max_rsize: 1048576
           max_wsize: 1048576
           tightly coupled: Yes
           notify_mask: 0x00000006 (Change, Delete)
               notify_type: Change (1)
               notify_type: Delete (2) [Main Opcode: GETDEVINFO (47)]

The MDS is mounted with IPv4. I can provide the full packet trace, if needed.

Regards,
  Tigran.

----- Original Message -----

Quoted 39 lines

From: "trondmy" <[email protected]>
To: "linux-nfs" <[email protected]>
Sent: Wednesday, 11 November, 2020 00:42:31
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data
channels

Quoted 8 lines · On Tue, 2020-11-10 at 18:18 -0500, [email protected] wrote:

From: Trond Myklebust <[email protected]>

Add support for connecting to the pNFS files/flexfiles data servers
through RDMA, assuming that the GETDEVICEINFO call advertises that
support.

Quoted 7 lines

v2: Fix layoutstats encoding for pNFS/flexfiles.
v3: Move most of the netid handling into the SUNRPC and RDMA modules.
    Fix up the mount code to benefit more from automated loading of
    SUNRPC transport modules.

Note that one cleanup that I did not perform, but which really could be
useful should we want to add more transport mechanisms, is to move the
code to parse stringified addresses (in particular IETF style "universal addresses") into the transport modules so that the actual
parsing of mount and pNFS transport info can be automatically extended
when new transport modules are added.

--
Trond Myklebust
Linux NFS client maintainer, Hammerspace
[email protected]

--
Trond Myklebust
CTO, Hammerspace Inc
4984 El Camino Real, Suite 208
Los Altos, CA 94022

www.hammer.space

#17 lore raw
Quoted 55 lines · Tigran

----- Original Message -----

Quoted 25 lines

From: "trondmy" <[email protected]>
To: "Tigran Mkrtchyan" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Friday, 13 November, 2020 23:45:00
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Quoted 5 lines · Tigran

After more testing, it looks like that client doesn't like notification bitmap:

Quoted 11 lines
[31576.789492] --> _nfs4_proc_getdeviceinfo
[31576.789503] --> nfs41_call_sync_prepare data->seq_server
000000001d17c43e
[31576.789507] --> nfs4_alloc_slot used_slots=0000
highest_used=4294967295 max_slots=16
[31576.789510] <-- nfs4_alloc_slot used_slots=0001 highest_used=0
slotid=0
[31576.789527] encode_sequence:
sessionid=2910695007:150995712:0:16777216 seqid=92462 slotid=0
max_slotid=0 cache_this=0
[31576.789991] decode_getdeviceinfo: unsupported notification
Quoted 8 lines

According to this, you appear to be returning a deviceinfo bitmap with
at least one non-zero entry that is not in the first 32-bit word. We
only ask for notifications for NOTIFY_DEVICEID4_CHANGE and NOTIFY_DEVICEID4_DELETE, so we only expect bitmap[0] to have non- zero
entries.

according to packet capture only bitmap[0] has non zero bits set. This is the reply of compound starting from nfs staus code, tag length and so on.

0000   00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 35 0010   00 00 00 00 5f ae 7d ad 00 03 00 09 00 00 00 00 0020   00 00 00 01 00 00 00 4c 00 00 00 00 00 00 00 0f 0030   00 00 00 0f 00 00 00 00 00 00 00 2f 00 00 00 00 0040   00 00 00 04 00 00 00 40 00 00 00 01 00 00 00 03 0050   74 63 70 00 00 00 00 16 31 33 31 2e 31 36 39 2e 0060   31 39 31 2e 31 34 33 2e 31 32 35 2e 34 39 00 00 0070   00 00 00 01 00 00 00 04 00 00 00 01 00 10 00 00 0080   00 10 00 00 00 00 00 01 00 00 00 02 00 00 00 06 0090   00 00 00 00

Quoted 6 lines

the last 12 bytes : bitmap size, bitmap[0], bitmap[1]

This part of code in the didn't change since 2010, and I have no issues to use 5.8 kernel. I am pretty sure, that tests with 5.9 did pass as expected. I will try to bisec it.

I don't think I've introduced this bug. I did not touch anything in the getdeviceinfo proc or XDR code.
Does the following patch help?

8<-------------------------------------------------------
From e92b2d4e39e91d379ec1147115820ab5dfe4c89a Mon Sep 17 00:00:00 2001
From: Trond Myklebust <[email protected]>
Date: Fri, 13 Nov 2020 21:42:16 -0500
Subject: [PATCH] NFSv4: Fix the alignment of page data in the getdeviceinfo
reply

We can fit the device_addr4 opaque data padding in the pages.

Fixes cf500bac8fd4 ("SUNRPC: Introduce rpc_prepare_reply_pages()")
---
 fs/nfs/nfs4xdr.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)
fs/nfs/nfs4xdr.c +10 -4
index c6dbfcae7517..c8714381d511 100644
@@ -3009,15 +3009,19 @@ static void nfs4_xdr_enc_getdeviceinfo(struct rpc_rqst *req,
 	struct compound_hdr hdr = {
 		.minorversion = nfs4_xdr_minorversion(&args->seq_args),
 	};
+	uint32_t replen;
 
 	encode_compound_hdr(xdr, req, &hdr);
 	encode_sequence(xdr, &args->seq_args, &hdr);
+
+	replen = hdr.replen + op_decode_hdr_maxsz;
+
 	encode_getdeviceinfo(xdr, args, &hdr);
 
-	/* set up reply kvec. Subtract notification bitmap max size (2)
-	 * so that notification bitmap is put in xdr_buf tail */
+	/* set up reply kvec. device_addr4 opaque data is read into the
+	 * pages */
 	rpc_prepare_reply_pages(req, args->pdev->pages, args->pdev->pgbase,
-				args->pdev->pglen, hdr.replen - 2);
+				args->pdev->pglen, replen + 2);
 	encode_nops(&hdr);
 }
 
@@ -5848,7 +5852,9 @@ static int decode_getdeviceinfo(struct xdr_stream *xdr,
 	 * and places the remaining xdr data in xdr_buf->tail
 	 */
 	pdev->mincount = be32_to_cpup(p);
-	if (xdr_read_pages(xdr, pdev->mincount) != pdev->mincount)
+	/* Calculate padding */
+	len = xdr_align_size(pdev->mincount);
+	if (xdr_read_pages(xdr, len) != len)
 		return -EIO;
 
 	/* Parse notification bitmap, verifying that it is zero. */
Signature · 8 lines
2.28.0



-- 
Trond Myklebust
Linux NFS client maintainer, Hammerspace
[email protected]
#18 lore raw

Hi Trond,

I am afraid, that the fix didn't work. I bisecting it....

Tigran.

----- Original Message -----

Quoted 136 lines

From: "trondmy" <[email protected]>
To: "Tigran Mkrtchyan" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Saturday, 14 November, 2020 15:29:01
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Quoted 55 lines · Tigran

----- Original Message -----

Quoted 25 lines

From: "trondmy" <[email protected]>
To: "Tigran Mkrtchyan" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Friday, 13 November, 2020 23:45:00
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Quoted 5 lines · Tigran

After more testing, it looks like that client doesn't like notification bitmap:

Quoted 11 lines
[31576.789492] --> _nfs4_proc_getdeviceinfo
[31576.789503] --> nfs41_call_sync_prepare data->seq_server
000000001d17c43e
[31576.789507] --> nfs4_alloc_slot used_slots=0000
highest_used=4294967295 max_slots=16
[31576.789510] <-- nfs4_alloc_slot used_slots=0001 highest_used=0
slotid=0
[31576.789527] encode_sequence:
sessionid=2910695007:150995712:0:16777216 seqid=92462 slotid=0
max_slotid=0 cache_this=0
[31576.789991] decode_getdeviceinfo: unsupported notification
Quoted 8 lines

According to this, you appear to be returning a deviceinfo bitmap with
at least one non-zero entry that is not in the first 32-bit word. We
only ask for notifications for NOTIFY_DEVICEID4_CHANGE and NOTIFY_DEVICEID4_DELETE, so we only expect bitmap[0] to have non- zero
entries.

according to packet capture only bitmap[0] has non zero bits set. This is the reply of compound starting from nfs staus code, tag length and so on.

0000   00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 35 0010   00 00 00 00 5f ae 7d ad 00 03 00 09 00 00 00 00 0020   00 00 00 01 00 00 00 4c 00 00 00 00 00 00 00 0f 0030   00 00 00 0f 00 00 00 00 00 00 00 2f 00 00 00 00 0040   00 00 00 04 00 00 00 40 00 00 00 01 00 00 00 03 0050   74 63 70 00 00 00 00 16 31 33 31 2e 31 36 39 2e 0060   31 39 31 2e 31 34 33 2e 31 32 35 2e 34 39 00 00 0070   00 00 00 01 00 00 00 04 00 00 00 01 00 10 00 00 0080   00 10 00 00 00 00 00 01 00 00 00 02 00 00 00 06 0090   00 00 00 00

Quoted 6 lines

the last 12 bytes : bitmap size, bitmap[0], bitmap[1]

This part of code in the didn't change since 2010, and I have no issues to use 5.8 kernel. I am pretty sure, that tests with 5.9 did pass as expected. I will try to bisec it.

I don't think I've introduced this bug. I did not touch anything in the getdeviceinfo proc or XDR code.
Does the following patch help?

8<-------------------------------------------------------
From e92b2d4e39e91d379ec1147115820ab5dfe4c89a Mon Sep 17 00:00:00 2001
From: Trond Myklebust <[email protected]>
Date: Fri, 13 Nov 2020 21:42:16 -0500
Subject: [PATCH] NFSv4: Fix the alignment of page data in the getdeviceinfo reply

We can fit the device_addr4 opaque data padding in the pages.

Fixes cf500bac8fd4 ("SUNRPC: Introduce rpc_prepare_reply_pages()")

---
fs/nfs/nfs4xdr.c | 14 ++++++++++----

1 file changed, 10 insertions(+), 4 deletions(-)
fs/nfs/nfs4xdr.c +10 -5
index c6dbfcae7517..c8714381d511 100644
@@ -3009,15 +3009,19 @@ static void nfs4_xdr_enc_getdeviceinfo(struct rpc_rqst
 *req,
 	struct compound_hdr hdr = {
 		.minorversion = nfs4_xdr_minorversion(&args->seq_args),
 	};
+	uint32_t replen;
 
 	encode_compound_hdr(xdr, req, &hdr);
 	encode_sequence(xdr, &args->seq_args, &hdr);
+
+	replen = hdr.replen + op_decode_hdr_maxsz;
+
 	encode_getdeviceinfo(xdr, args, &hdr);
 
-	/* set up reply kvec. Subtract notification bitmap max size (2)
-	 * so that notification bitmap is put in xdr_buf tail */
+	/* set up reply kvec. device_addr4 opaque data is read into the
+	 * pages */
 	rpc_prepare_reply_pages(req, args->pdev->pages, args->pdev->pgbase,
-				args->pdev->pglen, hdr.replen - 2);
+				args->pdev->pglen, replen + 2);
 	encode_nops(&hdr);
 }
@@ -5848,7 +5852,9 @@ static int decode_getdeviceinfo(struct xdr_stream *xdr,
 	 * and places the remaining xdr data in xdr_buf->tail
 	 */
 	pdev->mincount = be32_to_cpup(p);
-	if (xdr_read_pages(xdr, pdev->mincount) != pdev->mincount)
+	/* Calculate padding */
+	len = xdr_align_size(pdev->mincount);
+	if (xdr_read_pages(xdr, len) != len)
 		return -EIO;
 
 	/* Parse notification bitmap, verifying that it is zero. */
--

2.28.0

--
Trond Myklebust
Linux NFS client maintainer, Hammerspace
[email protected]

#19 lore raw

Here is the result:

$ git bisect bad 
c567552612ece787b178e3b147b5854ad422a836 is the first bad commit
commit c567552612ece787b178e3b147b5854ad422a836
Author: Anna Schumaker <[email protected]>
Date:   Wed May 28 13:41:22 2014 -0400

NFS: Add READ_PLUS data segment support

This patch adds client support for decoding a single NFS4_CONTENT_DATA segment returned by the server. This is the simplest implementation possible, since it does not account for any hole segments in the reply.

 fs/nfs/nfs42xdr.c         | 141 ++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfs/nfs4client.c       |   2 +
 fs/nfs/nfs4proc.c         |  43 +++++++++++++-
 fs/nfs/nfs4xdr.c          |   1 +
 include/linux/nfs4.h      |   2 +-
 include/linux/nfs_fs_sb.h |   1 +
 include/linux/nfs_xdr.h   |   2 +-
 7 files changed, 187 insertions(+), 5 deletions(-)

Regards,
Tigran.

----- Original Message -----

Quoted 152 lines

From: "Tigran Mkrtchyan" <[email protected]>
To: "trondmy" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Monday, 16 November, 2020 21:55:50
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Hi Trond,

I am afraid, that the fix didn't work. I bisecting it....

Tigran.

----- Original Message -----

Quoted 137 lines

From: "trondmy" <[email protected]>
To: "Tigran Mkrtchyan" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Saturday, 14 November, 2020 15:29:01
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Quoted 55 lines · Tigran

----- Original Message -----

Quoted 25 lines

From: "trondmy" <[email protected]>
To: "Tigran Mkrtchyan" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Friday, 13 November, 2020 23:45:00
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Quoted 5 lines · Tigran

After more testing, it looks like that client doesn't like notification bitmap:

Quoted 11 lines
[31576.789492] --> _nfs4_proc_getdeviceinfo
[31576.789503] --> nfs41_call_sync_prepare data->seq_server
000000001d17c43e
[31576.789507] --> nfs4_alloc_slot used_slots=0000
highest_used=4294967295 max_slots=16
[31576.789510] <-- nfs4_alloc_slot used_slots=0001 highest_used=0
slotid=0
[31576.789527] encode_sequence:
sessionid=2910695007:150995712:0:16777216 seqid=92462 slotid=0
max_slotid=0 cache_this=0
[31576.789991] decode_getdeviceinfo: unsupported notification
Quoted 8 lines

According to this, you appear to be returning a deviceinfo bitmap with
at least one non-zero entry that is not in the first 32-bit word. We
only ask for notifications for NOTIFY_DEVICEID4_CHANGE and NOTIFY_DEVICEID4_DELETE, so we only expect bitmap[0] to have non- zero
entries.

according to packet capture only bitmap[0] has non zero bits set. This is the reply of compound starting from nfs staus code, tag length and so on.

0000   00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 35 0010   00 00 00 00 5f ae 7d ad 00 03 00 09 00 00 00 00 0020   00 00 00 01 00 00 00 4c 00 00 00 00 00 00 00 0f 0030   00 00 00 0f 00 00 00 00 00 00 00 2f 00 00 00 00 0040   00 00 00 04 00 00 00 40 00 00 00 01 00 00 00 03 0050   74 63 70 00 00 00 00 16 31 33 31 2e 31 36 39 2e 0060   31 39 31 2e 31 34 33 2e 31 32 35 2e 34 39 00 00 0070   00 00 00 01 00 00 00 04 00 00 00 01 00 10 00 00 0080   00 10 00 00 00 00 00 01 00 00 00 02 00 00 00 06 0090   00 00 00 00

Quoted 6 lines

the last 12 bytes : bitmap size, bitmap[0], bitmap[1]

This part of code in the didn't change since 2010, and I have no issues to use 5.8 kernel. I am pretty sure, that tests with 5.9 did pass as expected. I will try to bisec it.

I don't think I've introduced this bug. I did not touch anything in the getdeviceinfo proc or XDR code.
Does the following patch help?

8<-------------------------------------------------------
From e92b2d4e39e91d379ec1147115820ab5dfe4c89a Mon Sep 17 00:00:00 2001
From: Trond Myklebust <[email protected]>
Date: Fri, 13 Nov 2020 21:42:16 -0500
Subject: [PATCH] NFSv4: Fix the alignment of page data in the getdeviceinfo reply

We can fit the device_addr4 opaque data padding in the pages.

Fixes cf500bac8fd4 ("SUNRPC: Introduce rpc_prepare_reply_pages()")

---
fs/nfs/nfs4xdr.c | 14 ++++++++++----

1 file changed, 10 insertions(+), 4 deletions(-)
fs/nfs/nfs4xdr.c +10 -5
index c6dbfcae7517..c8714381d511 100644
@@ -3009,15 +3009,19 @@ static void nfs4_xdr_enc_getdeviceinfo(struct rpc_rqst
 *req,
 	struct compound_hdr hdr = {
 		.minorversion = nfs4_xdr_minorversion(&args->seq_args),
 	};
+	uint32_t replen;
 
 	encode_compound_hdr(xdr, req, &hdr);
 	encode_sequence(xdr, &args->seq_args, &hdr);
+
+	replen = hdr.replen + op_decode_hdr_maxsz;
+
 	encode_getdeviceinfo(xdr, args, &hdr);
 
-	/* set up reply kvec. Subtract notification bitmap max size (2)
-	 * so that notification bitmap is put in xdr_buf tail */
+	/* set up reply kvec. device_addr4 opaque data is read into the
+	 * pages */
 	rpc_prepare_reply_pages(req, args->pdev->pages, args->pdev->pgbase,
-				args->pdev->pglen, hdr.replen - 2);
+				args->pdev->pglen, replen + 2);
 	encode_nops(&hdr);
 }
@@ -5848,7 +5852,9 @@ static int decode_getdeviceinfo(struct xdr_stream *xdr,
 	 * and places the remaining xdr data in xdr_buf->tail
 	 */
 	pdev->mincount = be32_to_cpup(p);
-	if (xdr_read_pages(xdr, pdev->mincount) != pdev->mincount)
+	/* Calculate padding */
+	len = xdr_align_size(pdev->mincount);
+	if (xdr_read_pages(xdr, len) != len)
 		return -EIO;
 
 	/* Parse notification bitmap, verifying that it is zero. */
--

2.28.0

--
Trond Myklebust
Linux NFS client maintainer, Hammerspace
[email protected]

#20 lore raw

I have added some debug info Ind got this:

decode_getdeviceinfo: layout type 4
decode_getdeviceinfo: layout size 64
decode_getdeviceinfo: layout notification bitmap size 1094994757

So it looks like that xdr_read_pages set xdr pointer to a wrong position and next read, which is bitmap size

5857         pdev->mincount = be32_to_cpup(p);
5858         dprintk("%s: layout size %u\n", __func__, pdev->mincount);
5859         if (xdr_read_pages(xdr, pdev->mincount) != pdev->mincount)
5860                 return -EIO;
5861 
5862         /* Parse notification bitmap, verifying that it is zero. */
5863         p = xdr_inline_decode(xdr, 4);
5864         if (unlikely(!p))
5865                 return -EIO;
5866         len = be32_to_cpup(p);
5867         dprintk("%s: layout notification bitmap size %u\n", __func__, len);
5868         if (len) {

5869 uint32_t i;

Tigran.

----- Original Message -----

Quoted 192 lines

From: "Tigran Mkrtchyan" <[email protected]>
To: "trondmy" <[email protected]>
Cc: "linux-nfs" <[email protected]>, "Anna Schumaker" <[email protected]>
Sent: Tuesday, 17 November, 2020 15:50:57
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Here is the result:

$ git bisect bad
c567552612ece787b178e3b147b5854ad422a836 is the first bad commit
commit c567552612ece787b178e3b147b5854ad422a836
Author: Anna Schumaker <[email protected]>
Date:   Wed May 28 13:41:22 2014 -0400

NFS: Add READ_PLUS data segment support

This patch adds client support for decoding a single NFS4_CONTENT_DATA segment returned by the server. This is the simplest implementation possible, since it does not account for any hole segments in the reply.

fs/nfs/nfs42xdr.c         | 141 ++++++++++++++++++++++++++++++++++++++++++++++
fs/nfs/nfs4client.c       |   2 +
fs/nfs/nfs4proc.c         |  43 +++++++++++++-
fs/nfs/nfs4xdr.c          |   1 +
include/linux/nfs4.h      |   2 +-
include/linux/nfs_fs_sb.h |   1 +
include/linux/nfs_xdr.h   |   2 +-
7 files changed, 187 insertions(+), 5 deletions(-)

Regards,
Tigran.

----- Original Message -----

Quoted 153 lines

From: "Tigran Mkrtchyan" <[email protected]>
To: "trondmy" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Monday, 16 November, 2020 21:55:50
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Hi Trond,

I am afraid, that the fix didn't work. I bisecting it....

Tigran.

----- Original Message -----

Quoted 137 lines

From: "trondmy" <[email protected]>
To: "Tigran Mkrtchyan" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Saturday, 14 November, 2020 15:29:01
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Quoted 55 lines · Tigran

----- Original Message -----

Quoted 25 lines

From: "trondmy" <[email protected]>
To: "Tigran Mkrtchyan" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Friday, 13 November, 2020 23:45:00
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Quoted 5 lines · Tigran

After more testing, it looks like that client doesn't like notification bitmap:

Quoted 11 lines
[31576.789492] --> _nfs4_proc_getdeviceinfo
[31576.789503] --> nfs41_call_sync_prepare data->seq_server
000000001d17c43e
[31576.789507] --> nfs4_alloc_slot used_slots=0000
highest_used=4294967295 max_slots=16
[31576.789510] <-- nfs4_alloc_slot used_slots=0001 highest_used=0
slotid=0
[31576.789527] encode_sequence:
sessionid=2910695007:150995712:0:16777216 seqid=92462 slotid=0
max_slotid=0 cache_this=0
[31576.789991] decode_getdeviceinfo: unsupported notification
Quoted 8 lines

According to this, you appear to be returning a deviceinfo bitmap with
at least one non-zero entry that is not in the first 32-bit word. We
only ask for notifications for NOTIFY_DEVICEID4_CHANGE and NOTIFY_DEVICEID4_DELETE, so we only expect bitmap[0] to have non- zero
entries.

according to packet capture only bitmap[0] has non zero bits set. This is the reply of compound starting from nfs staus code, tag length and so on.

0000   00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 35 0010   00 00 00 00 5f ae 7d ad 00 03 00 09 00 00 00 00 0020   00 00 00 01 00 00 00 4c 00 00 00 00 00 00 00 0f 0030   00 00 00 0f 00 00 00 00 00 00 00 2f 00 00 00 00 0040   00 00 00 04 00 00 00 40 00 00 00 01 00 00 00 03 0050   74 63 70 00 00 00 00 16 31 33 31 2e 31 36 39 2e 0060   31 39 31 2e 31 34 33 2e 31 32 35 2e 34 39 00 00 0070   00 00 00 01 00 00 00 04 00 00 00 01 00 10 00 00 0080   00 10 00 00 00 00 00 01 00 00 00 02 00 00 00 06 0090   00 00 00 00

Quoted 6 lines

the last 12 bytes : bitmap size, bitmap[0], bitmap[1]

This part of code in the didn't change since 2010, and I have no issues to use 5.8 kernel. I am pretty sure, that tests with 5.9 did pass as expected. I will try to bisec it.

I don't think I've introduced this bug. I did not touch anything in the getdeviceinfo proc or XDR code.
Does the following patch help?

8<-------------------------------------------------------
From e92b2d4e39e91d379ec1147115820ab5dfe4c89a Mon Sep 17 00:00:00 2001
From: Trond Myklebust <[email protected]>
Date: Fri, 13 Nov 2020 21:42:16 -0500
Subject: [PATCH] NFSv4: Fix the alignment of page data in the getdeviceinfo reply

We can fit the device_addr4 opaque data padding in the pages.

Fixes cf500bac8fd4 ("SUNRPC: Introduce rpc_prepare_reply_pages()")

---
fs/nfs/nfs4xdr.c | 14 ++++++++++----

1 file changed, 10 insertions(+), 4 deletions(-)
fs/nfs/nfs4xdr.c +10 -5
index c6dbfcae7517..c8714381d511 100644
@@ -3009,15 +3009,19 @@ static void nfs4_xdr_enc_getdeviceinfo(struct rpc_rqst
 *req,
 	struct compound_hdr hdr = {
 		.minorversion = nfs4_xdr_minorversion(&args->seq_args),
 	};
+	uint32_t replen;
 
 	encode_compound_hdr(xdr, req, &hdr);
 	encode_sequence(xdr, &args->seq_args, &hdr);
+
+	replen = hdr.replen + op_decode_hdr_maxsz;
+
 	encode_getdeviceinfo(xdr, args, &hdr);
 
-	/* set up reply kvec. Subtract notification bitmap max size (2)
-	 * so that notification bitmap is put in xdr_buf tail */
+	/* set up reply kvec. device_addr4 opaque data is read into the
+	 * pages */
 	rpc_prepare_reply_pages(req, args->pdev->pages, args->pdev->pgbase,
-				args->pdev->pglen, hdr.replen - 2);
+				args->pdev->pglen, replen + 2);
 	encode_nops(&hdr);
 }
@@ -5848,7 +5852,9 @@ static int decode_getdeviceinfo(struct xdr_stream *xdr,
 	 * and places the remaining xdr data in xdr_buf->tail
 	 */
 	pdev->mincount = be32_to_cpup(p);
-	if (xdr_read_pages(xdr, pdev->mincount) != pdev->mincount)
+	/* Calculate padding */
+	len = xdr_align_size(pdev->mincount);
+	if (xdr_read_pages(xdr, len) != len)
 		return -EIO;
 
 	/* Parse notification bitmap, verifying that it is zero. */
--

2.28.0

--
Trond Myklebust
Linux NFS client maintainer, Hammerspace
[email protected]

#21 lore raw

More investigation...

After xdr_read_pages call the xdr stream points to the beginning of RPC package and return the xid value on
the next read (which should be the size of the notification bitmap).

Regards,
Tigran.

----- Original Message -----

Quoted 228 lines

From: "Tigran Mkrtchyan" <[email protected]>
To: "trondmy" <[email protected]>
Cc: "linux-nfs" <[email protected]>, "Anna Schumaker" <[email protected]>
Sent: Thursday, 26 November, 2020 18:17:41
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

I have added some debug info Ind got this:

decode_getdeviceinfo: layout type 4
decode_getdeviceinfo: layout size 64
decode_getdeviceinfo: layout notification bitmap size 1094994757

So it looks like that xdr_read_pages set xdr pointer to a wrong position and next read, which is bitmap size

5857         pdev->mincount = be32_to_cpup(p);
5858         dprintk("%s: layout size %u\n", __func__, pdev->mincount);
5859         if (xdr_read_pages(xdr, pdev->mincount) != pdev->mincount)
5860                 return -EIO;
5861
5862         /* Parse notification bitmap, verifying that it is zero. */
5863         p = xdr_inline_decode(xdr, 4);
5864         if (unlikely(!p))
5865                 return -EIO;
5866         len = be32_to_cpup(p);
5867         dprintk("%s: layout notification bitmap size %u\n", __func__, len);
5868         if (len) {

5869 uint32_t i;

Tigran.

----- Original Message -----

Quoted 194 lines

From: "Tigran Mkrtchyan" <[email protected]>
To: "trondmy" <[email protected]>
Cc: "linux-nfs" <[email protected]>, "Anna Schumaker" <[email protected]>
Sent: Tuesday, 17 November, 2020 15:50:57
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Here is the result:

$ git bisect bad
c567552612ece787b178e3b147b5854ad422a836 is the first bad commit
commit c567552612ece787b178e3b147b5854ad422a836
Author: Anna Schumaker <[email protected]>
Date:   Wed May 28 13:41:22 2014 -0400

NFS: Add READ_PLUS data segment support

This patch adds client support for decoding a single NFS4_CONTENT_DATA segment returned by the server. This is the simplest implementation possible, since it does not account for any hole segments in the reply.

fs/nfs/nfs42xdr.c         | 141 ++++++++++++++++++++++++++++++++++++++++++++++
fs/nfs/nfs4client.c       |   2 +
fs/nfs/nfs4proc.c         |  43 +++++++++++++-
fs/nfs/nfs4xdr.c          |   1 +
include/linux/nfs4.h      |   2 +-
include/linux/nfs_fs_sb.h |   1 +
include/linux/nfs_xdr.h   |   2 +-
7 files changed, 187 insertions(+), 5 deletions(-)

Regards,
Tigran.

----- Original Message -----

Quoted 153 lines

From: "Tigran Mkrtchyan" <[email protected]>
To: "trondmy" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Monday, 16 November, 2020 21:55:50
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Hi Trond,

I am afraid, that the fix didn't work. I bisecting it....

Tigran.

----- Original Message -----

Quoted 137 lines

From: "trondmy" <[email protected]>
To: "Tigran Mkrtchyan" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Saturday, 14 November, 2020 15:29:01
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Quoted 55 lines · Tigran

----- Original Message -----

Quoted 25 lines

From: "trondmy" <[email protected]>
To: "Tigran Mkrtchyan" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Friday, 13 November, 2020 23:45:00
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Quoted 5 lines · Tigran

After more testing, it looks like that client doesn't like notification bitmap:

Quoted 11 lines
[31576.789492] --> _nfs4_proc_getdeviceinfo
[31576.789503] --> nfs41_call_sync_prepare data->seq_server
000000001d17c43e
[31576.789507] --> nfs4_alloc_slot used_slots=0000
highest_used=4294967295 max_slots=16
[31576.789510] <-- nfs4_alloc_slot used_slots=0001 highest_used=0
slotid=0
[31576.789527] encode_sequence:
sessionid=2910695007:150995712:0:16777216 seqid=92462 slotid=0
max_slotid=0 cache_this=0
[31576.789991] decode_getdeviceinfo: unsupported notification
Quoted 8 lines

According to this, you appear to be returning a deviceinfo bitmap with
at least one non-zero entry that is not in the first 32-bit word. We
only ask for notifications for NOTIFY_DEVICEID4_CHANGE and NOTIFY_DEVICEID4_DELETE, so we only expect bitmap[0] to have non- zero
entries.

according to packet capture only bitmap[0] has non zero bits set. This is the reply of compound starting from nfs staus code, tag length and so on.

0000   00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 35 0010   00 00 00 00 5f ae 7d ad 00 03 00 09 00 00 00 00 0020   00 00 00 01 00 00 00 4c 00 00 00 00 00 00 00 0f 0030   00 00 00 0f 00 00 00 00 00 00 00 2f 00 00 00 00 0040   00 00 00 04 00 00 00 40 00 00 00 01 00 00 00 03 0050   74 63 70 00 00 00 00 16 31 33 31 2e 31 36 39 2e 0060   31 39 31 2e 31 34 33 2e 31 32 35 2e 34 39 00 00 0070   00 00 00 01 00 00 00 04 00 00 00 01 00 10 00 00 0080   00 10 00 00 00 00 00 01 00 00 00 02 00 00 00 06 0090   00 00 00 00

Quoted 6 lines

the last 12 bytes : bitmap size, bitmap[0], bitmap[1]

This part of code in the didn't change since 2010, and I have no issues to use 5.8 kernel. I am pretty sure, that tests with 5.9 did pass as expected. I will try to bisec it.

I don't think I've introduced this bug. I did not touch anything in the getdeviceinfo proc or XDR code.
Does the following patch help?

8<-------------------------------------------------------
From e92b2d4e39e91d379ec1147115820ab5dfe4c89a Mon Sep 17 00:00:00 2001
From: Trond Myklebust <[email protected]>
Date: Fri, 13 Nov 2020 21:42:16 -0500
Subject: [PATCH] NFSv4: Fix the alignment of page data in the getdeviceinfo reply

We can fit the device_addr4 opaque data padding in the pages.

Fixes cf500bac8fd4 ("SUNRPC: Introduce rpc_prepare_reply_pages()")

---
fs/nfs/nfs4xdr.c | 14 ++++++++++----

1 file changed, 10 insertions(+), 4 deletions(-)
fs/nfs/nfs4xdr.c +10 -5
index c6dbfcae7517..c8714381d511 100644
@@ -3009,15 +3009,19 @@ static void nfs4_xdr_enc_getdeviceinfo(struct rpc_rqst
 *req,
 	struct compound_hdr hdr = {
 		.minorversion = nfs4_xdr_minorversion(&args->seq_args),
 	};
+	uint32_t replen;
 
 	encode_compound_hdr(xdr, req, &hdr);
 	encode_sequence(xdr, &args->seq_args, &hdr);
+
+	replen = hdr.replen + op_decode_hdr_maxsz;
+
 	encode_getdeviceinfo(xdr, args, &hdr);
 
-	/* set up reply kvec. Subtract notification bitmap max size (2)
-	 * so that notification bitmap is put in xdr_buf tail */
+	/* set up reply kvec. device_addr4 opaque data is read into the
+	 * pages */
 	rpc_prepare_reply_pages(req, args->pdev->pages, args->pdev->pgbase,
-				args->pdev->pglen, hdr.replen - 2);
+				args->pdev->pglen, replen + 2);
 	encode_nops(&hdr);
 }
@@ -5848,7 +5852,9 @@ static int decode_getdeviceinfo(struct xdr_stream *xdr,
 	 * and places the remaining xdr data in xdr_buf->tail
 	 */
 	pdev->mincount = be32_to_cpup(p);
-	if (xdr_read_pages(xdr, pdev->mincount) != pdev->mincount)
+	/* Calculate padding */
+	len = xdr_align_size(pdev->mincount);
+	if (xdr_read_pages(xdr, len) != len)
 		return -EIO;
 
 	/* Parse notification bitmap, verifying that it is zero. */
--

2.28.0

--
Trond Myklebust
Linux NFS client maintainer, Hammerspace
[email protected]

#22 lore raw

Actually, this was a co-incidence, which means that buf points to more or less a random place.

Tigran.

----- Original Message -----

Quoted 247 lines

From: "Tigran Mkrtchyan" <[email protected]>
To: "trondmy" <[email protected]>
Cc: "linux-nfs" <[email protected]>, "Anna Schumaker" <[email protected]>
Sent: Tuesday, 1 December, 2020 11:59:22
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

More investigation...

After xdr_read_pages call the xdr stream points to the beginning of RPC package and return the xid value on
the next read (which should be the size of the notification bitmap).

Regards,
Tigran.

----- Original Message -----

Quoted 230 lines

From: "Tigran Mkrtchyan" <[email protected]>
To: "trondmy" <[email protected]>
Cc: "linux-nfs" <[email protected]>, "Anna Schumaker" <[email protected]>
Sent: Thursday, 26 November, 2020 18:17:41
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

I have added some debug info Ind got this:

decode_getdeviceinfo: layout type 4
decode_getdeviceinfo: layout size 64
decode_getdeviceinfo: layout notification bitmap size 1094994757

So it looks like that xdr_read_pages set xdr pointer to a wrong position and next read, which is bitmap size

5857         pdev->mincount = be32_to_cpup(p);
5858         dprintk("%s: layout size %u\n", __func__, pdev->mincount);
5859         if (xdr_read_pages(xdr, pdev->mincount) != pdev->mincount)
5860                 return -EIO;
5861
5862         /* Parse notification bitmap, verifying that it is zero. */
5863         p = xdr_inline_decode(xdr, 4);
5864         if (unlikely(!p))
5865                 return -EIO;
5866         len = be32_to_cpup(p);
5867         dprintk("%s: layout notification bitmap size %u\n", __func__, len);
5868         if (len) {

5869 uint32_t i;

Tigran.

----- Original Message -----

Quoted 194 lines

From: "Tigran Mkrtchyan" <[email protected]>
To: "trondmy" <[email protected]>
Cc: "linux-nfs" <[email protected]>, "Anna Schumaker" <[email protected]>
Sent: Tuesday, 17 November, 2020 15:50:57
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Here is the result:

$ git bisect bad
c567552612ece787b178e3b147b5854ad422a836 is the first bad commit
commit c567552612ece787b178e3b147b5854ad422a836
Author: Anna Schumaker <[email protected]>
Date:   Wed May 28 13:41:22 2014 -0400

NFS: Add READ_PLUS data segment support

This patch adds client support for decoding a single NFS4_CONTENT_DATA segment returned by the server. This is the simplest implementation possible, since it does not account for any hole segments in the reply.

fs/nfs/nfs42xdr.c         | 141 ++++++++++++++++++++++++++++++++++++++++++++++
fs/nfs/nfs4client.c       |   2 +
fs/nfs/nfs4proc.c         |  43 +++++++++++++-
fs/nfs/nfs4xdr.c          |   1 +
include/linux/nfs4.h      |   2 +-
include/linux/nfs_fs_sb.h |   1 +
include/linux/nfs_xdr.h   |   2 +-
7 files changed, 187 insertions(+), 5 deletions(-)

Regards,
Tigran.

----- Original Message -----

Quoted 153 lines

From: "Tigran Mkrtchyan" <[email protected]>
To: "trondmy" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Monday, 16 November, 2020 21:55:50
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Hi Trond,

I am afraid, that the fix didn't work. I bisecting it....

Tigran.

----- Original Message -----

Quoted 137 lines

From: "trondmy" <[email protected]>
To: "Tigran Mkrtchyan" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Saturday, 14 November, 2020 15:29:01
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Quoted 55 lines · Tigran

----- Original Message -----

Quoted 25 lines

From: "trondmy" <[email protected]>
To: "Tigran Mkrtchyan" <[email protected]>
Cc: "linux-nfs" <[email protected]>
Sent: Friday, 13 November, 2020 23:45:00
Subject: Re: [PATCH v3 00/11] Add RDMA support to the pNFS file+flexfiles data channels

Quoted 5 lines · Tigran

After more testing, it looks like that client doesn't like notification bitmap:

Quoted 11 lines
[31576.789492] --> _nfs4_proc_getdeviceinfo
[31576.789503] --> nfs41_call_sync_prepare data->seq_server
000000001d17c43e
[31576.789507] --> nfs4_alloc_slot used_slots=0000
highest_used=4294967295 max_slots=16
[31576.789510] <-- nfs4_alloc_slot used_slots=0001 highest_used=0
slotid=0
[31576.789527] encode_sequence:
sessionid=2910695007:150995712:0:16777216 seqid=92462 slotid=0
max_slotid=0 cache_this=0
[31576.789991] decode_getdeviceinfo: unsupported notification
Quoted 8 lines

According to this, you appear to be returning a deviceinfo bitmap with
at least one non-zero entry that is not in the first 32-bit word. We
only ask for notifications for NOTIFY_DEVICEID4_CHANGE and NOTIFY_DEVICEID4_DELETE, so we only expect bitmap[0] to have non- zero
entries.

according to packet capture only bitmap[0] has non zero bits set. This is the reply of compound starting from nfs staus code, tag length and so on.

0000   00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 35 0010   00 00 00 00 5f ae 7d ad 00 03 00 09 00 00 00 00 0020   00 00 00 01 00 00 00 4c 00 00 00 00 00 00 00 0f 0030   00 00 00 0f 00 00 00 00 00 00 00 2f 00 00 00 00 0040   00 00 00 04 00 00 00 40 00 00 00 01 00 00 00 03 0050   74 63 70 00 00 00 00 16 31 33 31 2e 31 36 39 2e 0060   31 39 31 2e 31 34 33 2e 31 32 35 2e 34 39 00 00 0070   00 00 00 01 00 00 00 04 00 00 00 01 00 10 00 00 0080   00 10 00 00 00 00 00 01 00 00 00 02 00 00 00 06 0090   00 00 00 00

Quoted 6 lines

the last 12 bytes : bitmap size, bitmap[0], bitmap[1]

This part of code in the didn't change since 2010, and I have no issues to use 5.8 kernel. I am pretty sure, that tests with 5.9 did pass as expected. I will try to bisec it.

I don't think I've introduced this bug. I did not touch anything in the getdeviceinfo proc or XDR code.
Does the following patch help?

8<-------------------------------------------------------
From e92b2d4e39e91d379ec1147115820ab5dfe4c89a Mon Sep 17 00:00:00 2001
From: Trond Myklebust <[email protected]>
Date: Fri, 13 Nov 2020 21:42:16 -0500
Subject: [PATCH] NFSv4: Fix the alignment of page data in the getdeviceinfo reply

We can fit the device_addr4 opaque data padding in the pages.

Fixes cf500bac8fd4 ("SUNRPC: Introduce rpc_prepare_reply_pages()")

---
fs/nfs/nfs4xdr.c | 14 ++++++++++----

1 file changed, 10 insertions(+), 4 deletions(-)
fs/nfs/nfs4xdr.c +10 -5
index c6dbfcae7517..c8714381d511 100644
@@ -3009,15 +3009,19 @@ static void nfs4_xdr_enc_getdeviceinfo(struct rpc_rqst
 *req,
 	struct compound_hdr hdr = {
 		.minorversion = nfs4_xdr_minorversion(&args->seq_args),
 	};
+	uint32_t replen;
 
 	encode_compound_hdr(xdr, req, &hdr);
 	encode_sequence(xdr, &args->seq_args, &hdr);
+
+	replen = hdr.replen + op_decode_hdr_maxsz;
+
 	encode_getdeviceinfo(xdr, args, &hdr);
 
-	/* set up reply kvec. Subtract notification bitmap max size (2)
-	 * so that notification bitmap is put in xdr_buf tail */
+	/* set up reply kvec. device_addr4 opaque data is read into the
+	 * pages */
 	rpc_prepare_reply_pages(req, args->pdev->pages, args->pdev->pgbase,
-				args->pdev->pglen, hdr.replen - 2);
+				args->pdev->pglen, replen + 2);
 	encode_nops(&hdr);
 }
@@ -5848,7 +5852,9 @@ static int decode_getdeviceinfo(struct xdr_stream *xdr,
 	 * and places the remaining xdr data in xdr_buf->tail
 	 */
 	pdev->mincount = be32_to_cpup(p);
-	if (xdr_read_pages(xdr, pdev->mincount) != pdev->mincount)
+	/* Calculate padding */
+	len = xdr_align_size(pdev->mincount);
+	if (xdr_read_pages(xdr, len) != len)
 		return -EIO;
 
 	/* Parse notification bitmap, verifying that it is zero. */
--

2.28.0

--
Trond Myklebust
Linux NFS client maintainer, Hammerspace
[email protected]