Skip to contentSkip to conversation

SUNRPC: Create sysfs files for changing IP address

PATCH In review [email protected] 2 participants 7 messages Mar 12, 2021 – Mar 24, 2021

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

v3 cover SUNRPC: Create sysfs files for changing IP address

From: Anna Schumaker <[email protected]>

It's possible for an NFS server to go down but come back up with a different IP address. These patches provide a way for administrators to handle this issue by providing a new IP address for xprt sockets to connect to.

Chuck has suggested some ideas for future work that could also use this interface, such as:
- srcaddr: To move between network devices on the client
- type: "tcp", "rdma", "local"
- bound: 0 for autobind, or the result of the most recent rpcbind query
- connected: either true or false
- last: read-only timestamp of the last operation to use the transport
- device: A symlink to the physical network device

Changes in v3:
- Rename functions and objects to make future expansion easier
- Put files under /sys/kernel/sunrpc/client/ instead of /sys/kernel/sunrpc/net/, again for future expansions
- Clean up use of WARN_ON_ONCE() in xs_connect()
- Fix up locking, reference counting, and RCU usage
- Unconditionally create files so userspace tools don't need to guess what is supported (We return an error message now instead)

Changes in v2:
- Put files under /sys/kernel/sunrpc/ instead of /sys/net/sunrpc/
- Rename file from "address" to "dstaddr"

Thoughts?
Anna

Anna Schumaker (5):
sunrpc: Create a sunrpc directory under /sys/kernel/
sunrpc: Create a client/ subdirectory in the sunrpc sysfs
sunrpc: Create per-rpc_clnt sysfs kobjects
sunrpc: Prepare xs_connect() for taking NULL tasks
sunrpc: Create a per-rpc_clnt file for managing the destination IP
address

 include/linux/sunrpc/clnt.h |   1 +
 net/sunrpc/Makefile         |   2 +-
 net/sunrpc/clnt.c           |   5 +
 net/sunrpc/sunrpc_syms.c    |   8 ++
 net/sunrpc/sysfs.c          | 191 ++++++++++++++++++++++++++++++++++++
 net/sunrpc/sysfs.h          |  20 ++++
 net/sunrpc/xprtsock.c       |   2 +-
 7 files changed, 227 insertions(+), 2 deletions(-)
 create mode 100644 net/sunrpc/sysfs.c
 create mode 100644 net/sunrpc/sysfs.h
Signature · 1 line
2.29.2
#1 lore raw

PATCH v3 1/5 sunrpc: Create a sunrpc directory under /sys/kernel/

From: Anna Schumaker <[email protected]>

This is where we'll put per-rpc_client related files

---
v3: Rename kset object to rpc_sunrpc_kset
v2: Move directory from /sys/net/sunrpc/ to /sys/kernel/sunrpc/

---
 net/sunrpc/Makefile      |  2 +-
 net/sunrpc/sunrpc_syms.c |  8 ++++++++
 net/sunrpc/sysfs.c       | 20 ++++++++++++++++++++
 net/sunrpc/sysfs.h       | 11 +++++++++++
 4 files changed, 40 insertions(+), 1 deletion(-)
 create mode 100644 net/sunrpc/sysfs.c
 create mode 100644 net/sunrpc/sysfs.h
net/sunrpc/Makefile +1 -1
index 9488600451e8..1c8de397d6ad 100644
@@ -12,7 +12,7 @@ sunrpc-y := clnt.o xprt.o socklib.o xprtsock.o sched.o \
 	    auth.o auth_null.o auth_unix.o \
 	    svc.o svcsock.o svcauth.o svcauth_unix.o \
 	    addr.o rpcb_clnt.o timer.o xdr.o \
-	    sunrpc_syms.o cache.o rpc_pipe.o \
+	    sunrpc_syms.o cache.o rpc_pipe.o sysfs.o \
 	    svc_xprt.o \
 	    xprtmultipath.o
 sunrpc-$(CONFIG_SUNRPC_DEBUG) += debugfs.o
net/sunrpc/sunrpc_syms.c +8 -0
index 236fadc4a439..3b57efc692ec 100644
@@ -24,6 +24,7 @@
 #include <linux/sunrpc/xprtsock.h>
 
 #include "sunrpc.h"
+#include "sysfs.h"
 #include "netns.h"
 
 unsigned int sunrpc_net_id;
@@ -103,6 +104,10 @@ init_sunrpc(void)
 	if (err)
 		goto out4;
 
+	err = rpc_sysfs_init();
+	if (err)
+		goto out5;
+
 	sunrpc_debugfs_init();
 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
 	rpc_register_sysctl();
@@ -111,6 +116,8 @@ init_sunrpc(void)
 	init_socket_xprt();	/* clnt sock transport */
 	return 0;
 
+out5:
+	unregister_rpc_pipefs();
 out4:
 	unregister_pernet_subsys(&sunrpc_net_ops);
 out3:
@@ -124,6 +131,7 @@ init_sunrpc(void)
 static void __exit
 cleanup_sunrpc(void)
 {
+	rpc_sysfs_exit();
 	rpc_cleanup_clids();
 	rpcauth_remove_module();
 	cleanup_socket_xprt();
net/sunrpc/sysfs.c new file +20 -0
new file mode 100644
index 000000000000..27eda180ac5e
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2020 Anna Schumaker <[email protected]>
+ */
+#include <linux/kobject.h>
+
+static struct kset *rpc_sunrpc_kset;
+
+int rpc_sysfs_init(void)
+{
+	rpc_sunrpc_kset = kset_create_and_add("sunrpc", NULL, kernel_kobj);
+	if (!rpc_sunrpc_kset)
+		return -ENOMEM;
+	return 0;
+}
+
+void rpc_sysfs_exit(void)
+{
+	kset_unregister(rpc_sunrpc_kset);
+}
net/sunrpc/sysfs.h new file +11 -0
new file mode 100644
index 000000000000..c9d9918e87ee
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2020 Anna Schumaker <[email protected]>
+ */
+#ifndef __SUNRPC_SYSFS_H
+#define __SUNRPC_SYSFS_H
+
+extern int rpc_sysfs_init(void);
+extern void rpc_sysfs_exit(void);
+
+#endif
Signature · 1 line
2.29.2
#2 lore raw

PATCH v3 2/5 sunrpc: Create a client/ subdirectory in the sunrpc sysfs

In reply to [email protected]

From: Anna Schumaker <[email protected]>

For network namespace separation.

---
v3: Rename net/ directory to client/
Rename rpc_client_kobj to rpc_sunrpc_client_kobj s/netns/sysfs/

---
 net/sunrpc/sysfs.c | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
net/sunrpc/sysfs.c +41 -0
index 27eda180ac5e..350b2b1628e7 100644
@@ -2,19 +2,60 @@
 /*
  * Copyright (c) 2020 Anna Schumaker <[email protected]>
  */
+#include <linux/sunrpc/clnt.h>
 #include <linux/kobject.h>
 
 static struct kset *rpc_sunrpc_kset;
+static struct kobject *rpc_sunrpc_client_kobj;
+
+static void rpc_sysfs_object_release(struct kobject *kobj)
+{
+	kfree(kobj);
+}
+
+static const struct kobj_ns_type_operations *rpc_sysfs_object_child_ns_type(
+		struct kobject *kobj)
+{
+	return &net_ns_type_operations;
+}
+
+static struct kobj_type rpc_sysfs_object_type = {
+	.release = rpc_sysfs_object_release,
+	.sysfs_ops = &kobj_sysfs_ops,
+	.child_ns_type = rpc_sysfs_object_child_ns_type,
+};
+
+static struct kobject *rpc_sysfs_object_alloc(const char *name,
+		struct kset *kset, struct kobject *parent)
+{
+	struct kobject *kobj;
+	kobj = kzalloc(sizeof(*kobj), GFP_KERNEL);
+	if (kobj) {
+		kobj->kset = kset;
+		if (kobject_init_and_add(kobj, &rpc_sysfs_object_type,
+					parent, "%s", name) == 0)
+			return kobj;
+		kobject_put(kobj);
+	}
+	return NULL;
+}
 
 int rpc_sysfs_init(void)
 {
 	rpc_sunrpc_kset = kset_create_and_add("sunrpc", NULL, kernel_kobj);
 	if (!rpc_sunrpc_kset)
 		return -ENOMEM;
+	rpc_sunrpc_client_kobj = rpc_sysfs_object_alloc("client", rpc_sunrpc_kset, NULL);
+	if (!rpc_sunrpc_client_kobj) {
+		kset_unregister(rpc_sunrpc_kset);
+		rpc_sunrpc_client_kobj = NULL;
+		return -ENOMEM;
+	}
 	return 0;
 }
 
 void rpc_sysfs_exit(void)
 {
+	kobject_put(rpc_sunrpc_client_kobj);
 	kset_unregister(rpc_sunrpc_kset);
 }
Signature · 1 line
2.29.2
#3 lore raw

PATCH v3 3/5 sunrpc: Create per-rpc_clnt sysfs kobjects

In reply to [email protected]

From: Anna Schumaker <[email protected]>

These will eventually have files placed under them for sysfs operations.

---
v3: s/netns/sysfs/

---
 include/linux/sunrpc/clnt.h |  1 +
 net/sunrpc/clnt.c           |  5 ++++
 net/sunrpc/sysfs.c          | 60 +++++++++++++++++++++++++++++++++++++
 net/sunrpc/sysfs.h          |  8 +++++
 4 files changed, 74 insertions(+)
include/linux/sunrpc/clnt.h +1 -0
index 02e7a5863d28..503653720e18 100644
@@ -71,6 +71,7 @@ struct rpc_clnt {
 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
 	struct dentry		*cl_debugfs;	/* debugfs directory */
 #endif
+	void			*cl_sysfs;	/* sysfs directory */
 	/* cl_work is only needed after cl_xpi is no longer used,
 	 * and that are of similar size
 	 */
net/sunrpc/clnt.c +5 -0
index 612f0a641f4c..ceb8d19d4cb4 100644
@@ -41,6 +41,7 @@
 #include <trace/events/sunrpc.h>
 
 #include "sunrpc.h"
+#include "sysfs.h"
 #include "netns.h"
 
 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
@@ -300,6 +301,7 @@ static int rpc_client_register(struct rpc_clnt *clnt,
 	int err;
 
 	rpc_clnt_debugfs_register(clnt);
+	rpc_sysfs_client_setup(clnt, net);
 
 	pipefs_sb = rpc_get_sb_net(net);
 	if (pipefs_sb) {
@@ -327,6 +329,7 @@ static int rpc_client_register(struct rpc_clnt *clnt,
 out:
 	if (pipefs_sb)
 		rpc_put_sb_net(net);
+	rpc_sysfs_client_destroy(clnt);
 	rpc_clnt_debugfs_unregister(clnt);
 	return err;
 }
@@ -733,6 +736,7 @@ int rpc_switch_client_transport(struct rpc_clnt *clnt,
 
 	rpc_unregister_client(clnt);
 	__rpc_clnt_remove_pipedir(clnt);
+	rpc_sysfs_client_destroy(clnt);
 	rpc_clnt_debugfs_unregister(clnt);
 
 	/*
@@ -879,6 +883,7 @@ static void rpc_free_client_work(struct work_struct *work)
 	 * so they cannot be called in rpciod, so they are handled separately
 	 * here.
 	 */
+	rpc_sysfs_client_destroy(clnt);
 	rpc_clnt_debugfs_unregister(clnt);
 	rpc_free_clid(clnt);
 	rpc_clnt_remove_pipedir(clnt);
net/sunrpc/sysfs.c +60 -0

74 lines not on this page · raw message

net/sunrpc/sysfs.h +8 -0
index c9d9918e87ee..443679d71f38 100644
@@ -5,7 +5,15 @@
 #ifndef __SUNRPC_SYSFS_H
 #define __SUNRPC_SYSFS_H
 
+struct rpc_sysfs_client {
+	struct kobject kobject;
+	struct net *net;
+};
+
 extern int rpc_sysfs_init(void);
 extern void rpc_sysfs_exit(void);
 
+void rpc_sysfs_client_setup(struct rpc_clnt *clnt, struct net *net);
+void rpc_sysfs_client_destroy(struct rpc_clnt *clnt);
+
 #endif
Signature · 1 line
2.29.2
#4 lore raw

PATCH v3 4/5 sunrpc: Prepare xs_connect() for taking NULL tasks

In reply to [email protected]

From: Anna Schumaker <[email protected]>

We won't have a task structure when we go to change IP addresses, so check for one before calling the WARN_ON() to avoid crashing.

---
v3: Clean up WARN_ON_ONCE() to avoid a new if

---
 net/sunrpc/xprtsock.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
net/sunrpc/xprtsock.c +1 -1
index e35760f238a4..eb8df10aac9c 100644
@@ -2310,7 +2310,7 @@ static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task)
 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
 	unsigned long delay = 0;
 
-	WARN_ON_ONCE(!xprt_lock_connect(xprt, task, transport));
+	WARN_ON_ONCE(task && !xprt_lock_connect(xprt, task, transport));
 
 	if (transport->sock != NULL) {
 		dprintk("RPC:       xs_connect delayed xprt %p for %lu "
Signature · 1 line
2.29.2
#5 lore raw

PATCH v3 5/5 sunrpc: Create a per-rpc_clnt file for managing the destination IP address

In reply to [email protected]

From: Anna Schumaker <[email protected]>

Reading the file displays the current destination address, and writing to it allows users to change the address.

And since we're using IP here, restrict to only creating sysfs files for TCP and RDMA connections.

---
v3: Fix suspicious RCU usage warnings
Fix up xprt locking and reference counting s/netns/sysfs/
Unconditionally create files instead of looking at protocol type
during client setup (this makes it easier for userspace tools)
Add a newline character to the buffer returned in "show" handler
v2: Change filename and related functions from "address" to "dstaddr"
Combine patches for reading and writing to this file

---
 net/sunrpc/sysfs.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++
 net/sunrpc/sysfs.h |  1 +
 2 files changed, 71 insertions(+)
net/sunrpc/sysfs.c +70 -0
index abfe8c0b3108..1ae15c4729c0 100644
@@ -3,6 +3,8 @@
  * Copyright (c) 2020 Anna Schumaker <[email protected]>
  */
 #include <linux/sunrpc/clnt.h>
+#include <linux/sunrpc/addr.h>
+#include <linux/sunrpc/xprtsock.h>
 #include <linux/kobject.h>
 #include "sysfs.h"
 
@@ -55,6 +57,64 @@ int rpc_sysfs_init(void)
 	return 0;
 }
 
+static inline struct rpc_xprt *rpc_sysfs_client_kobj_get_xprt(struct kobject *kobj)
+{
+	struct rpc_sysfs_client *c = container_of(kobj,
+				struct rpc_sysfs_client, kobject);
+	struct rpc_xprt *xprt;
+
+	rcu_read_lock();
+	xprt = xprt_get(rcu_dereference(c->clnt->cl_xprt));
+	rcu_read_unlock();
+	return xprt;
+}
+
+static ssize_t rpc_sysfs_dstaddr_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	struct rpc_xprt *xprt = rpc_sysfs_client_kobj_get_xprt(kobj);
+	ssize_t ret;
+
+	if (!xprt)
+		return 0;
+	if (!(xprt->prot & (IPPROTO_TCP | XPRT_TRANSPORT_RDMA)))
+		ret = sprintf(buf, "Not Supported");
+	else
+		ret = rpc_ntop((struct sockaddr *)&xprt->addr, buf, PAGE_SIZE);
+	buf[ret] = '\n';
+	xprt_put(xprt);
+	return ret + 1;
+}
+
+static ssize_t rpc_sysfs_dstaddr_store(struct kobject *kobj,
+		struct kobj_attribute *attr, const char *buf, size_t count)
+{
+	struct rpc_xprt *xprt = rpc_sysfs_client_kobj_get_xprt(kobj);
+	struct sockaddr *saddr;
+	int port;
+
+	if (!xprt)
+		return 0;
+	if (!(xprt->prot & (IPPROTO_TCP | XPRT_TRANSPORT_RDMA))) {
+		xprt_put(xprt);
+		return -ENOTSUPP;
+	}
+
+	wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE);
+	saddr = (struct sockaddr *)&xprt->addr;
+	port = rpc_get_port(saddr);
+
+	kfree(xprt->address_strings[RPC_DISPLAY_ADDR]);
+	xprt->address_strings[RPC_DISPLAY_ADDR] = kstrndup(buf, count - 1, GFP_KERNEL);
+	xprt->addrlen = rpc_pton(xprt->xprt_net, buf, count - 1, saddr, sizeof(*saddr));
+	rpc_set_port(saddr, port);
+
+	xprt->ops->connect(xprt, NULL);
+	clear_bit(XPRT_LOCKED, &xprt->state);
+	xprt_put(xprt);
+	return count;
+}
+
 static void rpc_sysfs_client_release(struct kobject *kobj)
 {
 	struct rpc_sysfs_client *c;
@@ -68,8 +128,17 @@ static const void *rpc_sysfs_client_namespace(struct kobject *kobj)
 	return container_of(kobj, struct rpc_sysfs_client, kobject)->net;
 }
 
+static struct kobj_attribute rpc_sysfs_client_dstaddr = __ATTR(dstaddr,
+		0644, rpc_sysfs_dstaddr_show, rpc_sysfs_dstaddr_store);
+
+static struct attribute *rpc_sysfs_client_attrs[] = {
+	&rpc_sysfs_client_dstaddr.attr,
+	NULL,
+};
+
 static struct kobj_type rpc_sysfs_client_type = {
 	.release = rpc_sysfs_client_release,
+	.default_attrs = rpc_sysfs_client_attrs,
 	.sysfs_ops = &kobj_sysfs_ops,
 	.namespace = rpc_sysfs_client_namespace,
 };
@@ -104,6 +173,7 @@ void rpc_sysfs_client_setup(struct rpc_clnt *clnt, struct net *net)
 	rpc_client = rpc_sysfs_client_alloc(rpc_sunrpc_client_kobj, net, clnt->cl_clid);
 	if (rpc_client) {
 		clnt->cl_sysfs = rpc_client;
+		rpc_client->clnt = clnt;
 		kobject_uevent(&rpc_client->kobject, KOBJ_ADD);
 	}
 }
net/sunrpc/sysfs.h +1 -0
index 443679d71f38..5093f93a83cb 100644
@@ -8,6 +8,7 @@
 struct rpc_sysfs_client {
 	struct kobject kobject;
 	struct net *net;
+	struct rpc_clnt *clnt;
 };
 
 extern int rpc_sysfs_init(void);
Signature · 1 line
2.29.2
#6 lore raw
In reply to [email protected]
Quoted 26 lines · On Fri, Mar 12, 2021 at 4:19 PM <[email protected]> wrote:

From: Anna Schumaker <[email protected]>

It's possible for an NFS server to go down but come back up with a different IP address. These patches provide a way for administrators to handle this issue by providing a new IP address for xprt sockets to connect to.

Chuck has suggested some ideas for future work that could also use this interface, such as:
- srcaddr: To move between network devices on the client
- type: "tcp", "rdma", "local"
- bound: 0 for autobind, or the result of the most recent rpcbind query
- connected: either true or false
- last: read-only timestamp of the last operation to use the transport
- device: A symlink to the physical network device

Changes in v3:
- Rename functions and objects to make future expansion easier
- Put files under /sys/kernel/sunrpc/client/ instead of /sys/kernel/sunrpc/net/, again for future expansions
- Clean up use of WARN_ON_ONCE() in xs_connect()
- Fix up locking, reference counting, and RCU usage
- Unconditionally create files so userspace tools don't need to guess what is supported (We return an error message now instead)

Quoted 5 lines

Changes in v2:
- Put files under /sys/kernel/sunrpc/ instead of /sys/net/sunrpc/
- Rename file from "address" to "dstaddr"

Thoughts?

Reviewed-by/Tested-by this version. Works OK for me.

I would like to note that the interface doesn't or rather perhaps cannot do any error checking. So if the "user" were to echo a nonsensical data into the sysfs (echo foobar > <sysfspath>), that breaks the existing connection. However, if a proper IP were to be entered to correct it, things will go back to normal.

Quoted 25 lines

Anna

Anna Schumaker (5):
sunrpc: Create a sunrpc directory under /sys/kernel/
sunrpc: Create a client/ subdirectory in the sunrpc sysfs
sunrpc: Create per-rpc_clnt sysfs kobjects
sunrpc: Prepare xs_connect() for taking NULL tasks
sunrpc: Create a per-rpc_clnt file for managing the destination IP
address

 include/linux/sunrpc/clnt.h |   1 +
 net/sunrpc/Makefile         |   2 +-
 net/sunrpc/clnt.c           |   5 +
 net/sunrpc/sunrpc_syms.c    |   8 ++
 net/sunrpc/sysfs.c          | 191 ++++++++++++++++++++++++++++++++++++
 net/sunrpc/sysfs.h          |  20 ++++
 net/sunrpc/xprtsock.c       |   2 +-
 7 files changed, 227 insertions(+), 2 deletions(-)
 create mode 100644 net/sunrpc/sysfs.c
 create mode 100644 net/sunrpc/sysfs.h

--
2.29.2