Skip to contentSkip to conversation

fuse: add generic file store

RFC PATCH In review Peng Tao 6 participants 18 messages Jun 1, 2021 – Jun 24, 2021

Subsystems (2)
lore
Share & tools
lore

Peng Tao

author
#0 lore raw

RFC PATCH 1/1 fuse: add generic file store

Add a generic file store that userspace can save/restore any open file descriptor. These file descriptors can be managed by different applications not just the same user space application.

A possible use case is fuse fd passthrough being developed by Alessio Balsini [1] where underlying file system fd can be saved in this file store.

Another possible use case is user space application live upgrade and failover (upon panic etc.). Currently during userspace live upgrade and failover, open file descriptors usually have to be saved seprately in a different management process with AF_UNIX sendmsg.

But it causes chicken and egg problem and such management process needs to support live upgrade and failover as well. With a generic file store in the kernel, application live upgrade and failover no longer require such management process to hold reference for their open file descriptors.

This is an RFC to see if the approach makes sense to upstream and it can be tested with the following C programe.

Why FUSE?
- Because we are trying to solve FUSE fd passthrough and FUSE daemon live upgrade.

Why global IDR rather than per fuse connnection one?
- Because for live upgrade new process, we don't have a valid fuse connection in the first place.

Missing cleanup method in case user space messes up?
- We can limit the number of saved FDs and hey it is RFC ;).

[1] https://lore.kernel.org/lkml/[email protected]/
--------

/#include <sys/types.h>
/#include <sys/stat.h>
/#include <fcntl.h>
/#include <unistd.h>
/#include <stdlib.h>
/#include <stdio.h>
/#include <stdbool.h>
/#include <string.h>
/#include <sys/ioctl.h>
/#include <stdint.h>

/#define FUSE_DEV_IOC_SAVE_FD _IOWR(229, 1, uint32_t)
/#define FUSE_DEV_IOC_RESTORE_FD _IOWR(229, 2, uint32_t) /#define FUSE_DEV_IOC_REMOVE_FD _IOW(229, 3, uint32_t)

static const char *FUSEDEV = "/dev/fuse";
int fuse_fd;

void show_help(char *pname)

{
	fprintf(stdout, "%s [-sh] [-d <id>] [-f <filename>] [-r <id>]\n", pname);
	fprintf(stdout, "\t-s open <filename>(default foobar) and save its fd\n");
	fprintf(stdout, "\t-r <id> restore fd identified by id\n");
	fprintf(stdout, "\t-f <filename> set filename to save\n");
	fprintf(stdout, "\t-d <id> delete id identified by id\n");
	fprintf(stdout, "\t-h show this help\n");
}

int open_save_fd(char *name)

{
	int fd;

	fd = open(name, O_RDWR | O_CREAT, 0666);
	if (fd < 0)
		return fd;
	if (ioctl(fuse_fd, FUSE_DEV_IOC_SAVE_FD, &fd) < 0)
		return -1;

	return fd;
}

int restore_fd(int id)

{
	if (ioctl(fuse_fd, FUSE_DEV_IOC_RESTORE_FD, &id) < 0)
		return -1;
	return id;
}

int delete_fd(int id)

{
	return ioctl(fuse_fd, FUSE_DEV_IOC_REMOVE_FD, &id);
}

int read_file(int fd)

{
	char buf[1024];
	int ret;

	lseek(fd, 0, SEEK_SET);
	memset(buf, 0, sizeof(buf));
	fprintf(stdout, "file content:\n");
	while ((ret = read(fd, buf, sizeof(buf))) > 0) {
		fprintf(stdout, "%s", buf);
	}
	return 0;
}

int main(int argc, char *argv[])

{
	bool save = false, restore = false;
	int fd, opt, ret, id, restored_fd;
	char *name = "foobar";

	fuse_fd = open(FUSEDEV, O_RDWR);
	if (fuse_fd < 0) {
		fprintf(stderr, "failed to open fuse device\n");
		return -1;
	}

	while ((opt = getopt(argc, argv, "shd:f:r:")) != -1) {
		switch (opt) {
		case 's':
			save = true;
			break;
		case 'r':
			id = atoi(optarg);
			restored_fd = restore_fd(id);
			if (restored_fd < 0)
				fprintf(stderr, "failed to restore fd with id %d\n", id);
			else
				fprintf(stdout, "restored file fd %d with id %d\n", restored_fd, id);
			restore = true;
			break;
		case 'd':
			id = atoi(optarg);
			ret = delete_fd(id);
			if (ret < 0)
				fprintf(stderr, "failed to delete fd with id %d\n", id);
			else
				fprintf(stdout, "deleted file id %d\n", id);
		case 'f':
			name = strdup(optarg);
			break;
		case 'h':
		default:
			show_help(argv[0]);
			return 0;
		}
	}
	if (save) {
		ret = open_save_fd(name);
		if (ret < 0)
			fprintf(stderr, "failed to open save fd with filename %s\n", name);
		else
			fprintf(stdout, "saved file id %d\n", ret);
	}
	if (restore && restored_fd > 0) {
		ret = read_file(restored_fd);
		if (ret < 0)
			fprintf(stderr, "failed to read_file with fd %d\n", restored_fd);
	}

	return 0;
}
---
 fs/fuse/dev.c             | 31 ++++++++++++++---
 fs/fuse/fuse_i.h          |  5 +++
 fs/fuse/inode.c           | 85 +++++++++++++++++++++++++++++++++++++++++++++++
 include/uapi/linux/fuse.h |  3 ++
 4 files changed, 119 insertions(+), 5 deletions(-)
fs/fuse/dev.c +26 -5
index a5ceccc..dfce809 100644
@@ -2229,15 +2229,14 @@ static int fuse_device_clone(struct fuse_conn *fc, struct file *new)
 static long fuse_dev_ioctl(struct file *file, unsigned int cmd,
 			   unsigned long arg)
 {
-	int res;
-	int oldfd;
+	int res = -EFAULT;
+	int fd;
 	struct fuse_dev *fud = NULL;
 
 	switch (cmd) {
 	case FUSE_DEV_IOC_CLONE:
-		res = -EFAULT;
-		if (!get_user(oldfd, (__u32 __user *)arg)) {
-			struct file *old = fget(oldfd);
+		if (!get_user(fd, (__u32 __user *)arg)) {
+			struct file *old = fget(fd);
 
 			res = -EINVAL;
 			if (old) {
@@ -2258,6 +2257,28 @@ static long fuse_dev_ioctl(struct file *file, unsigned int cmd,
 			}
 		}
 		break;
+	case FUSE_DEV_IOC_SAVE_FD:
+		if (!get_user(fd, (__u32 __user *) arg)) {
+			res = fuse_filp_save(fd);
+			if (res > 0) {
+				res = put_user(res, (__u32 __user *) arg);
+			}
+		}
+		break;
+	case FUSE_DEV_IOC_RESTORE_FD:
+		if (!get_user(fd, (__u32 __user *) arg)) {
+			res = fuse_filp_restore(fd);
+			if (res > 0) {
+				res = put_user(res, (__u32 __user *) arg);
+			}
+		}
+		break;
+	case FUSE_DEV_IOC_REMOVE_FD:
+		if (!get_user(fd, (__u32 __user *) arg)) {
+			fuse_filp_remove(fd);
+			res = 0;
+		}
+		break;
 	default:
 		res = -ENOTTY;
 		break;
fs/fuse/fuse_i.h +5 -0
index 7e463e2..1060391 100644
@@ -1259,4 +1259,9 @@ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid,
 void fuse_file_release(struct inode *inode, struct fuse_file *ff,
 		       unsigned int open_flags, fl_owner_t id, bool isdir);
 
+/* fuse fd store management */
+int fuse_filp_save(int fd);
+int fuse_filp_restore(int id);
+void fuse_filp_remove(int id);
+
 #endif /* _FS_FUSE_I_H */
fs/fuse/inode.c +85 -0

109 lines not on this page · raw message

include/uapi/linux/fuse.h +3 -0
index 271ae90..d12a5c2 100644
@@ -919,6 +919,9 @@ struct fuse_notify_retrieve_in {
 /* Device ioctls: */
 #define FUSE_DEV_IOC_MAGIC		229
 #define FUSE_DEV_IOC_CLONE		_IOR(FUSE_DEV_IOC_MAGIC, 0, uint32_t)
+#define FUSE_DEV_IOC_SAVE_FD		_IOWR(FUSE_DEV_IOC_MAGIC, 1, uint32_t)
+#define FUSE_DEV_IOC_RESTORE_FD		_IOWR(FUSE_DEV_IOC_MAGIC, 2, uint32_t)
+#define FUSE_DEV_IOC_REMOVE_FD		_IOW(FUSE_DEV_IOC_MAGIC, 3, uint32_t)
 
 struct fuse_lseek_in {
 	uint64_t	fh;
Signature · 1 line
1.8.3.1
#1 lore raw
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in https://git-scm.com/docs/git-format-patch]

Hi Peng,

[FYI, it's a private test report for your RFC patch.]
[auto build test WARNING on v5.13-rc4]
[also build test WARNING on next-20210601]
[cannot apply to fuse/for-next]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Peng-Tao/fuse-add-generic-file-store/20210601-170405
base:    8124c8a6b35386f73523d27eacb71b5364a68c4c
config: x86_64-randconfig-s022-20210601 (attached as .config)
compiler: gcc-9 (Debian 9.3.0-22) 9.3.0
reproduce:
        # apt-get install sparse
        # sparse version: v0.6.3-341-g8af24329-dirty
        # https://github.com/0day-ci/linux/commit/7f4d572f64348205578780f5a994676ae3a8bd78
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review Peng-Tao/fuse-add-generic-file-store/20210601-170405
        git checkout 7f4d572f64348205578780f5a994676ae3a8bd78
        # save the attached .config to linux build tree
        make W=1 C=1 CF='-fdiagnostic-prefix -D__CHECK_ENDIAN__' W=1 ARCH=x86_64 

If you fix the issue, kindly add following tag as appropriate

sparse warnings: (new ones prefixed by >>)

>> fs/fuse/inode.c:36:1: sparse: sparse: symbol 'fuse_filp_store' was not declared. Should it be static?
>> fs/fuse/inode.c:37:1: sparse: sparse: symbol 'fuse_filp_store_mutex' was not declared. Should it be static?

Please review and possibly fold the followup patch.

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

#2 lore raw

[RFC PATCH] fuse: fuse_filp_store can be static

In reply to Peng Tao
fs/fuse/inode.c:36:1: warning: symbol 'fuse_filp_store' was not declared. Should it be static? fs/fuse/inode.c:37:1: warning: symbol 'fuse_filp_store_mutex' was not declared. Should it be static?

fs/fuse/inode.c:36:1: warning: symbol 'fuse_filp_store' was not declared. Should it be static? fs/fuse/inode.c:37:1: warning: symbol 'fuse_filp_store_mutex' was not declared. Should it be static?

---
 inode.c |    4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
fs/fuse/inode.c +2 -2
index b95c5976da630..bdd3212956ed7 100644
@@ -33,8 +33,8 @@ static struct kmem_cache *fuse_inode_cachep;
 struct list_head fuse_conn_list;
 DEFINE_MUTEX(fuse_mutex);
 
-DEFINE_IDR(fuse_filp_store);
-DEFINE_MUTEX(fuse_filp_store_mutex);
+static DEFINE_IDR(fuse_filp_store);
+static DEFINE_MUTEX(fuse_filp_store_mutex);
 
 static int set_global_limit(const char *val, const struct kernel_param *kp);
 
#3 lore raw
In reply to Peng Tao
Quoted 29 lines · Peng Tao

Add a generic file store that userspace can save/restore any open file descriptor. These file descriptors can be managed by different applications not just the same user space application.

A possible use case is fuse fd passthrough being developed by Alessio Balsini [1] where underlying file system fd can be saved in this file store.

Another possible use case is user space application live upgrade and failover (upon panic etc.). Currently during userspace live upgrade and failover, open file descriptors usually have to be saved seprately in a different management process with AF_UNIX sendmsg.

But it causes chicken and egg problem and such management process needs to support live upgrade and failover as well. With a generic file store in the kernel, application live upgrade and failover no longer require such management process to hold reference for their open file descriptors.

This is an RFC to see if the approach makes sense to upstream and it can be tested with the following C programe.

Why FUSE?
- Because we are trying to solve FUSE fd passthrough and FUSE daemon live upgrade.

Why global IDR rather than per fuse connnection one?
- Because for live upgrade new process, we don't have a valid fuse connection in the first place.

Quoted 8 lines

Missing cleanup method in case user space messes up?
- We can limit the number of saved FDs and hey it is RFC ;).

[1] https://lore.kernel.org/lkml/[email protected]/
--------

[...]

Hi Peng,

This is a cool feature indeed.

I guess we also want to ensure that restoring an FD can only be performed by a trusted FUSE daemon, and not any other process attached to /dev/fuse. Maybe adding some permission checks?

I also see that multiple restores can be done on the same FD, is that intended? Shouldn't the IDR entry be removed once restored?

As far as I understand, the main use case is to be able to replace a FUSE daemon with another, preserving the opened lower file system files. How would user space handle the unmounting of the old FUSE file system and mounting of the new one?
I wonder if something can be done with a pair of ioctls similar to FUSE_DEV_IOC_CLONE to transfer the FUSE connection from the old to the new FUSE daemon. Maybe either the IDR or some other container to store the files that are intended to be preserved can be put in fuse_conn instead of keeping it global.

Does it make sense?

Thanks,
Alessio

#4 lore raw
Quoted 34 lines · Alessio Balsini
Quoted 32 lines · Peng Tao

Add a generic file store that userspace can save/restore any open file descriptor. These file descriptors can be managed by different applications not just the same user space application.

A possible use case is fuse fd passthrough being developed by Alessio Balsini [1] where underlying file system fd can be saved in this file store.

Another possible use case is user space application live upgrade and failover (upon panic etc.). Currently during userspace live upgrade and failover, open file descriptors usually have to be saved seprately in a different management process with AF_UNIX sendmsg.

But it causes chicken and egg problem and such management process needs to support live upgrade and failover as well. With a generic file store in the kernel, application live upgrade and failover no longer require such management process to hold reference for their open file descriptors.

This is an RFC to see if the approach makes sense to upstream and it can be tested with the following C programe.

Why FUSE?
- Because we are trying to solve FUSE fd passthrough and FUSE daemon live upgrade.

Why global IDR rather than per fuse connnection one?
- Because for live upgrade new process, we don't have a valid fuse connection in the first place.

Missing cleanup method in case user space messes up?
- We can limit the number of saved FDs and hey it is RFC ;).

Quoted 8 lines
Quoted 5 lines

[1] https://lore.kernel.org/lkml/[email protected]/
--------

[...]

Hi Peng,

Hi Alessio,

Quoted 7 lines

This is a cool feature indeed.

I guess we also want to ensure that restoring an FD can only be performed by a trusted FUSE daemon, and not any other process attached to /dev/fuse. Maybe adding some permission checks?

The idea is to allow any daemon capable of opening /dev/fuse to be able to restore an FD. I don't quite get which permissions do you like to check? SYS_ADMIN?

Quoted 3 lines

I also see that multiple restores can be done on the same FD, is that intended? Shouldn't the IDR entry be removed once restored?

In a crash recovery scenario, if the kernel destroys the IDR once an FD is restored, and the recovering daemon panics again, the FD is lost forever. So I would prefer to keep it in the kernel until explicit FD removal.

Quoted 4 lines

As far as I understand, the main use case is to be able to replace a FUSE daemon with another, preserving the opened lower file system files. How would user space handle the unmounting of the old FUSE file system and mounting of the new one?

It can call FUSE_DEV_IOC_REMOVE_FD before or after unmounting the old FUSE file system. Either way, the last closer of the FUSE connection FD would actually close the FUSE connection.

Quoted 8 lines

I wonder if something can be done with a pair of ioctls similar to FUSE_DEV_IOC_CLONE to transfer the FUSE connection from the old to the new FUSE daemon. Maybe either the IDR or some other container to store the files that are intended to be preserved can be put in fuse_conn instead of keeping it global.

Does it make sense?

It makes sense at first glance since obviously it helps IDR cleaning up as we can do it on a per fuse_conn basis. But giving it a second thought, how do we preserve the FUSE connection fd representing the same fuse_conn itself? We need to do it because we want to handle FUSE daemon crash recovery cases. Maybe we can have something like:
1. use a tag to uniquely identify a fuse conn (as being done for virtio-fs)
2. in each of these SAVE/GET/REMOVE ioctls, it takes a tag argument so FDs are kept locally to the specified fuse_conn
3. add FUSE_DEV_IOC_TRANSFER ioctl to transfer ownership of a saved FD to a new userspace daemon. It can be seen as a combination of GET and REMOVE ioctls

Cheers,
Tao

Signature · 1 line
Into Sth. Rich & Strange
#5 lore raw
In reply to Alessio Balsini
Quoted 3 lines · Alessio Balsini
Quoted 3 lines

A possible use case is fuse fd passthrough being developed by Alessio Balsini [1] where underlying file system fd can be saved in this file store.

oh, this could be quite what I'm currently intending do implement (just considered 9p instead of fuse as it looks simpler to me):

I'd like the server being able to directly send an already opened fd to the client (in response to it calling open()), quite like we can send fd's via unix sockets.

The primary use case of that is sending already prepared fd's (eg. an active network connection, locked-down file fd, a device that the client can't open himself, etc).

Is that what you're working on ?

--mtx

Signature · 8 lines
---
Hinweis: unverschlüsselte E-Mails können leicht abgehört und manipuliert
werden ! Für eine vertrauliche Kommunikation senden Sie bitte ihren
GPG/PGP-Schlüssel zu.
---
Enrico Weigelt, metux IT consult
Free software and Linux embedded engineering
[email protected] -- +49-151-27565287
#6 lore raw
Quoted 14 lines · metux IT consult
Quoted 3 lines · Alessio Balsini
Quoted 3 lines

A possible use case is fuse fd passthrough being developed by Alessio Balsini [1] where underlying file system fd can be saved in this file store.

oh, this could be quite what I'm currently intending do implement (just considered 9p instead of fuse as it looks simpler to me):

I'd like the server being able to directly send an already opened fd to the client (in response to it calling open()), quite like we can send fd's via unix sockets.

Quoted 5 lines

The primary use case of that is sending already prepared fd's (eg. an active network connection, locked-down file fd, a device that the client can't open himself, etc).

Is that what you're working on ?

If the server and client run on the same kernel, then yes, the current RFC supports your use case as well, E.g.,
1. the server opens a file, saves the FD to the kernel, and passes the IDR to the client.
2. the client retrieves the FD from the kernel

Does it match your use case?

Cheers,
Tao

Signature · 1 line
Into Sth. Rich & Strange
#7 lore raw
Quoted 11 lines · Peng Tao
Quoted 11 lines

oh, this could be quite what I'm currently intending do implement (just considered 9p instead of fuse as it looks simpler to me):

I'd like the server being able to directly send an already opened fd to the client (in response to it calling open()), quite like we can send fd's via unix sockets.

The primary use case of that is sending already prepared fd's (eg. an active network connection, locked-down file fd, a device that the client can't open himself, etc).

Quoted 8 lines
Quoted 1 line

Is that what you're working on ?

If the server and client run on the same kernel, then yes, the current RFC supports your use case as well, E.g.,
1. the server opens a file, saves the FD to the kernel, and passes the IDR to the client.
2. the client retrieves the FD from the kernel

Does it match your use case?

Seems that's exactly what I'm looking for :)

Could you perhaps give a little example code how it looks in userland ?

--mtx

Signature · 8 lines
---
Hinweis: unverschlüsselte E-Mails können leicht abgehört und manipuliert
werden ! Für eine vertrauliche Kommunikation senden Sie bitte ihren
GPG/PGP-Schlüssel zu.
---
Enrico Weigelt, metux IT consult
Free software and Linux embedded engineering
[email protected] -- +49-151-27565287
#8 lore raw
Quoted 21 lines · metux IT consult
Quoted 18 lines · Peng Tao
Quoted 7 lines

oh, this could be quite what I'm currently intending do implement (just considered 9p instead of fuse as it looks simpler to me):

I'd like the server being able to directly send an already opened fd to the client (in response to it calling open()), quite like we can send fd's via unix sockets.

Quoted 5 lines

The primary use case of that is sending already prepared fd's (eg. an active network connection, locked-down file fd, a device that the client can't open himself, etc).

Is that what you're working on ?

If the server and client run on the same kernel, then yes, the current RFC supports your use case as well, E.g.,
1. the server opens a file, saves the FD to the kernel, and passes the IDR to the client.
2. the client retrieves the FD from the kernel

Quoted 5 lines
Quoted 1 line

Does it match your use case?

Seems that's exactly what I'm looking for :)

Could you perhaps give a little example code how it looks in userland ?

The initial RFC mail in the thread has a userspace example code. Does it make sense to you?

Cheers,
Tao

Signature · 1 line
Into Sth. Rich & Strange
#9 lore raw

On 08.06.21 14:41, Peng Tao wrote:

Hi,

Quoted 2 lines

The initial RFC mail in the thread has a userspace example code. Does it make sense to you?

Sorry, I had missed that, now found it.

There're some things I don't quite understand:

* it just stores fd's I don't see anything where it is actually returned to some open() operation.
* the store is machine wide global - everybody uses the same number space, dont see any kind of access conrol ... how about security ?

I don't believe that just storing the fd's somewhere is really helpful for that purpose - the fuse server shall be able to reply the open() request with an fd, which then is directly transferred to the client.

--mtx

Signature · 8 lines
---
Hinweis: unverschlüsselte E-Mails können leicht abgehört und manipuliert
werden ! Für eine vertrauliche Kommunikation senden Sie bitte ihren
GPG/PGP-Schlüssel zu.
---
Enrico Weigelt, metux IT consult
Free software and Linux embedded engineering
[email protected] -- +49-151-27565287
#10 lore raw
Quoted 5 lines · metux IT consult

On 08.06.21 14:41, Peng Tao wrote:

Hi,

Hi,

Quoted 9 lines
Quoted 2 lines

The initial RFC mail in the thread has a userspace example code. Does it make sense to you?

Sorry, I had missed that, now found it.

There're some things I don't quite understand:

* it just stores fd's I don't see anything where it is actually returned to some open() operation.

The FUSE_DEV_IOC_RESTORE_FD ioctl returns the opened fd to a different process.

Quoted 3 lines

* the store is machine wide global - everybody uses the same number space, dont see any kind of access conrol ... how about security ?

The idea is that anyone capable of opening /dev/fuse can retrieve the FD.

Quoted 4 lines

I don't believe that just storing the fd's somewhere is really helpful for that purpose - the fuse server shall be able to reply the open() request with an fd, which then is directly transferred to the client.

Could you describe your use case a bit? How does your client talk to your server? Through open syscall or through some process-to-process RPC calls?

Cheers,
Tao

Signature · 1 line
Into Sth. Rich & Strange
#11 lore raw
Quoted 4 lines · Peng Tao
Quoted 3 lines

* it just stores fd's I don't see anything where it is actually returned to some open() operation.

The FUSE_DEV_IOC_RESTORE_FD ioctl returns the opened fd to a different process.

So, just open() a file on a fuse fs can't restore the fd directly (instead of opening a new file) ? If that's the case, that would mean, userland has to take very special actions in order to get it. Right ?

Quoted 5 lines
Quoted 3 lines

* the store is machine wide global - everybody uses the same number space, dont see any kind of access conrol ... how about security ?

The idea is that anyone capable of opening /dev/fuse can retrieve the FD.

Quoted 7 lines
Quoted 4 lines

I don't believe that just storing the fd's somewhere is really helpful for that purpose - the fuse server shall be able to reply the open() request with an fd, which then is directly transferred to the client.

Could you describe your use case a bit? How does your client talk to your server? Through open syscall or through some process-to-process RPC calls?

I'd like to write synthetic file systems (file servers) that allows certain unprivileged processes (in some confined environment) directly open()ing prepared file descriptors (e.g. devices, sockets, etc) that it isn't allowed to open directly (but the server obviously is). Those fds could be prepared in any ways (eg. sealed, seek'ed, already connected sockets, etc).

The client thinks it just open()'s a normal file, but actually gets some fd prepared elsewhere.

--mtx

Signature · 8 lines
---
Hinweis: unverschlüsselte E-Mails können leicht abgehört und manipuliert
werden ! Für eine vertrauliche Kommunikation senden Sie bitte ihren
GPG/PGP-Schlüssel zu.
---
Enrico Weigelt, metux IT consult
Free software and Linux embedded engineering
[email protected] -- +49-151-27565287
#12 lore raw
Quoted 11 lines · metux IT consult
Quoted 4 lines · Peng Tao
Quoted 3 lines

* it just stores fd's I don't see anything where it is actually returned to some open() operation.

The FUSE_DEV_IOC_RESTORE_FD ioctl returns the opened fd to a different process.

So, just open() a file on a fuse fs can't restore the fd directly (instead of opening a new file) ? If that's the case, that would mean, userland has to take very special actions in order to get it. Right ?

Yes.

Quoted 21 lines
Quoted 6 lines
Quoted 4 lines

* the store is machine wide global - everybody uses the same number space, dont see any kind of access conrol ... how about security ?

The idea is that anyone capable of opening /dev/fuse can retrieve the FD.

Quoted 7 lines
Quoted 4 lines

I don't believe that just storing the fd's somewhere is really helpful for that purpose - the fuse server shall be able to reply the open() request with an fd, which then is directly transferred to the client.

Could you describe your use case a bit? How does your client talk to your server? Through open syscall or through some process-to-process RPC calls?

I'd like to write synthetic file systems (file servers) that allows certain unprivileged processes (in some confined environment) directly open()ing prepared file descriptors (e.g. devices, sockets, etc) that it isn't allowed to open directly (but the server obviously is). Those fds could be prepared in any ways (eg. sealed, seek'ed, already connected sockets, etc).

Quoted 3 lines

The client thinks it just open()'s a normal file, but actually gets some fd prepared elsewhere.

Oh, nop, that is not how the current RFC works. I see two gaps:
1. /dev/fuse is not accessible to all processes by default
2. open() syscall doesn't take enough arguments to tell the kernel which file's fd it wants.

It seems that a proper solution to your use case is to:
1. extend the open() syscall to take a flag like FOPEN_FUSE_OPEN_FD (I agree it's a bad name;)
2. FUSE kernel passes such a flag to fuse daemon
3. FUSE userspace daemon opens the file in the underlying file system, store it to a kernel FD store, then return its IDR in the reply to FUSE_OPEN API
4. FUSE kernel looks up underlying FD with the IDR, install it in the calling process FD table, and return the new FD to the application

Is it what you want? It looks doable and is indeed an extension to the current RFC.

Cheers,
Tao

Signature · 1 line
Into Sth. Rich & Strange
#13 lore raw
Quoted 4 lines · Peng Tao
Quoted 3 lines

So, just open() a file on a fuse fs can't restore the fd directly (instead of opening a new file) ? If that's the case, that would mean, userland has to take very special actions in order to get it. Right ?

Yes.

Then, it's not at all what I'm looking for :(

Quoted 2 lines

Oh, nop, that is not how the current RFC works. I see two gaps:
1. /dev/fuse is not accessible to all processes by default

it shouldn't necessary at all - in my use case.

Quoted 2 lines

2. open() syscall doesn't take enough arguments to tell the kernel which file's fd it wants.

open() only works on a path name - that's exactly what I need.

Could you please tell more what your use case is actually about ? Still struggling what you're actually going to achieve.

Just keeping fd's open while a server restarts ?
If that's what you want, I see much wider use far outside of fuse, and that might call for some more generic approach - something like Plan9's /srv filesystem.

Quoted 3 lines

It seems that a proper solution to your use case is to:
1. extend the open() syscall to take a flag like FOPEN_FUSE_OPEN_FD (I agree it's a bad name;)

But that would still require changes in my userland. Something I do not want per definition - it should work transparently. The application just knows some file name (it might be even expecting common device names, but put inside its own mount ns), nothing more, and no need to change the application itself.

Quoted 6 lines

2. FUSE kernel passes such a flag to fuse daemon
3. FUSE userspace daemon opens the file in the underlying file system, store it to a kernel FD store, then return its IDR in the reply to FUSE_OPEN API
4. FUSE kernel looks up underlying FD with the IDR, install it in the calling process FD table, and return the new FD to the application

Does FUSE actually manipulate the process' fd table directly, while in the open() callback ?

--mtx

Signature · 8 lines
---
Hinweis: unverschlüsselte E-Mails können leicht abgehört und manipuliert
werden ! Für eine vertrauliche Kommunikation senden Sie bitte ihren
GPG/PGP-Schlüssel zu.
---
Enrico Weigelt, metux IT consult
Free software and Linux embedded engineering
[email protected] -- +49-151-27565287
#14 lore raw
Quoted 19 lines · metux IT consult
Quoted 4 lines · Peng Tao
Quoted 3 lines

So, just open() a file on a fuse fs can't restore the fd directly (instead of opening a new file) ? If that's the case, that would mean, userland has to take very special actions in order to get it. Right ?

Yes.

Then, it's not at all what I'm looking for :(

Quoted 2 lines

Oh, nop, that is not how the current RFC works. I see two gaps:
1. /dev/fuse is not accessible to all processes by default

it shouldn't necessary at all - in my use case.

Quoted 2 lines

2. open() syscall doesn't take enough arguments to tell the kernel which file's fd it wants.

open() only works on a path name - that's exactly what I need.

Quoted 8 lines

Could you please tell more what your use case is actually about ? Still struggling what you're actually going to achieve.

Just keeping fd's open while a server restarts ?
If that's what you want, I see much wider use far outside of fuse, and that might call for some more generic approach - something like Plan9's /srv filesystem.

1. keeping FDs across userspace restart
2. help save FD in the FUSE fd passthrough use case as implemented by Alessio Balsini

Quoted 17 lines
Quoted 3 lines

It seems that a proper solution to your use case is to:
1. extend the open() syscall to take a flag like FOPEN_FUSE_OPEN_FD (I agree it's a bad name;)

But that would still require changes in my userland. Something I do not want per definition - it should work transparently. The application just knows some file name (it might be even expecting common device names, but put inside its own mount ns), nothing more, and no need to change the application itself.

Quoted 6 lines

2. FUSE kernel passes such a flag to fuse daemon
3. FUSE userspace daemon opens the file in the underlying file system, store it to a kernel FD store, then return its IDR in the reply to FUSE_OPEN API
4. FUSE kernel looks up underlying FD with the IDR, install it in the calling process FD table, and return the new FD to the application

Quoted 2 lines

Does FUSE actually manipulate the process' fd table directly, while in the open() callback ?

hmm, you are right. The open() callback cannot install FD from there. So in order for your use case to work, the VFS layer needs to be changed to transparently replace an empty file struct with another file struct that is prepared by the file system somewhere else. It is really beyond the current RFC patch's scope IMHO.

Cheers,
Tao

Cheers,
Tao
--
Into Sth. Rich & Strange

#15 lore raw
Quoted 6 lines · Peng Tao
Quoted 5 lines

Just keeping fd's open while a server restarts ?
If that's what you want, I see much wider use far outside of fuse, and that might call for some more generic approach - something like Plan9's /srv filesystem.

1. keeping FDs across userspace restart

if application needs to be rewritten for that anyways, there're other ways to achieve this, w/o touching the kernel at all - exec() doesn't automatically close fd's (unless they're opened w/ O_CLOEXEC)

Quoted 2 lines

2. help save FD in the FUSE fd passthrough use case as implemented by Alessio Balsini

you mean this one ?

https://lore.kernel.org/lkml/[email protected]

I fail to see why an extra fd store within the fuse device is necessary for that - I'd just let the fuse daemon(s) reply the open request with the fd it already holds.

I'd hate to run into situations where even killing all processes holding some file open leads to a situation where it remains open inside the kernel, thus blocking e.g. unmounting. I already see operators getting very angy ... :o

by the way: alessio's approach is limited to simple read/write operations anyways - other operations like ioctl() don't seem to work easily that way.

and for the creds switching: I tend to believe that cases where a fs or device looks at the calling process' creds in operations on an already open fd, it's most likely a bad implementation.

yes, some legacy drivers actually do check for CAP_SYS_ADMIN e.g. for low level hardware configuration (e.g. IO and IRQ on ISA bus), but I wonder whether these are use at all in the our use cases and should be ever allowed to non-root.

do you have any case where you really need to use the opener's creds ? (after the fd is already open)

Quoted 8 lines
Quoted 2 lines

Does FUSE actually manipulate the process' fd table directly, while in the open() callback ?

hmm, you are right. The open() callback cannot install FD from there. So in order for your use case to work, the VFS layer needs to be changed to transparently replace an empty file struct with another file struct that is prepared by the file system somewhere else. It is really beyond the current RFC patch's scope IMHO.

Exactly. That's where I'm struggling right now. Yet have to find out whether I could just copy from one struct file into another (probably some refcnt'ing required). And that still has some drawback: fd state like file position won't be shared.

I've been thinking about changing the vfs_open() chain so that it doesn't pass in an existing/prepared struct file, but instead returns one, which is allocated further down the chain, right before the fs' open operation is called. Then we could add another variant that returns struct file. If the new one is present, it will be called, otherwise a new struct file is allocated, the old variant is called on the newly allocated one, and finally return this one.

this is a bigger job to do ...

--mtx

Signature · 8 lines
---
Hinweis: unverschlüsselte E-Mails können leicht abgehört und manipuliert
werden ! Für eine vertrauliche Kommunikation senden Sie bitte ihren
GPG/PGP-Schlüssel zu.
---
Enrico Weigelt, metux IT consult
Free software and Linux embedded engineering
[email protected] -- +49-151-27565287
#16 lore raw
Quoted 8 lines · metux IT consult
Quoted 5 lines · Peng Tao
Quoted 5 lines

Just keeping fd's open while a server restarts ?
If that's what you want, I see much wider use far outside of fuse, and that might call for some more generic approach - something like Plan9's /srv filesystem.

Quoted 5 lines
Quoted 1 line

1. keeping FDs across userspace restart

if application needs to be rewritten for that anyways, there're other ways to achieve this, w/o touching the kernel at all - exec() doesn't automatically close fd's (unless they're opened w/ O_CLOEXEC)

Or application recovery after panic ;)

Quoted 11 lines
Quoted 3 lines

2. help save FD in the FUSE fd passthrough use case as implemented by Alessio Balsini

you mean this one ?

https://lore.kernel.org/lkml/[email protected]

I fail to see why an extra fd store within the fuse device is necessary for that - I'd just let the fuse daemon(s) reply the open request with the fd it already holds.

Alessio already has a similar implementation in his patchset. The RPC patch tries to make it generic and thus usable for other use cases like fuse daemon upgrade and panic-recovery.b

Quoted 5 lines

I'd hate to run into situations where even killing all processes holding some file open leads to a situation where it remains open inside the kernel, thus blocking e.g. unmounting. I already see operators getting very angy ... :o

This is really a different design approach. The idea is to keep an FD active beyond the lifetime of a running process so that we can do panic recovery. Alessio's patchset has similar side effect in some corner cases and this RFC patch makes it a semantic promise. Whether ops like it would really depend on what they want.

Quoted 9 lines

by the way: alessio's approach is limited to simple read/write operations anyways - other operations like ioctl() don't seem to work easily that way.

and for the creds switching: I tend to believe that cases where a fs or device looks at the calling process' creds in operations on an already open fd, it's most likely a bad implementation.

I agree but I understand the rationale as well. A normal FUSE read/write uses FUSE daemon creds so the semantics are the same. Otherwise as you outline below, we'd have to go through all the read/write callbacks to make sure none of them is checking process creds.

Quoted 30 lines

yes, some legacy drivers actually do check for CAP_SYS_ADMIN e.g. for low level hardware configuration (e.g. IO and IRQ on ISA bus), but I wonder whether these are use at all in the our use cases and should be ever allowed to non-root.

do you have any case where you really need to use the opener's creds ? (after the fd is already open)

Quoted 8 lines
Quoted 2 lines

Does FUSE actually manipulate the process' fd table directly, while in the open() callback ?

hmm, you are right. The open() callback cannot install FD from there. So in order for your use case to work, the VFS layer needs to be changed to transparently replace an empty file struct with another file struct that is prepared by the file system somewhere else. It is really beyond the current RFC patch's scope IMHO.

Exactly. That's where I'm struggling right now. Yet have to find out whether I could just copy from one struct file into another (probably some refcnt'ing required). And that still has some drawback: fd state like file position won't be shared.

I've been thinking about changing the vfs_open() chain so that it doesn't pass in an existing/prepared struct file, but instead returns one, which is allocated further down the chain, right before the fs' open operation is called. Then we could add another variant that returns struct file. If the new one is present, it will be called, otherwise a new struct file is allocated, the old variant is called on the newly allocated one, and finally return this one.

Quoted 2 lines

this is a bigger job to do ...

Agreed.

Cheers,
Tao

Signature · 1 line
Into Sth. Rich & Strange
#17 lore raw
Quoted 1 line · Peng Tao

Or application recovery after panic ;)

Okay, thats a different scenario. But, of course, the application needs to have the fds already registered before it crashes. This gives two options:

a) always store them right after opening and do it's own garbage collection (e.g. on each close() call). - expensive and complex
b) construct it in a way that evem on critical signals (eg. sigsegv, sigbus) a signal handler can still store the fds somewhere (eg. using statically allocated memory and extra stack) - tricky

OTOH, i'd try to construct in a way that a crash of some master process (that can hold the fds even across restarts) is very unlikely to crash, since it doesn't do much more than that (plus spawning workers).

By the way, if you just wanna store fd's - i'm working on a more generic
solution: an Plan9-style srvfs. It's a file system that stores aready opened fd's and on open() gives your that fd (instead of a new one).

My previous experimental implementation did that indirectly by bridging all operations under the hood (tedious to synchronize the whole file state), but I'm now taking a fresh start w/ adding some "file boxing" mechanism to the kernel (patches not ready for submission):

* a file systems's open operation can put a pointer to another struct file into the struct file it's operating on -- that's what I called a "boxed file".
* the places (actually two) that actually create new struct file's and call into the open chain (eg. through vfs_open() etc) then do the unboxing -- if there's a boxed file, they fetch it out and drop the just newly fd.

Quoted 3 lines

Alessio already has a similar implementation in his patchset. The RPC patch tries to make it generic and thus usable for other use cases like fuse daemon upgrade and panic-recovery.b

I believe this shouldn't be some fuse specific thing. And we certainly have to make sure that it can't be abused for dos'ing the machine. Not sure whether that should be accounted to a namespace or cgroup.

Quoted 9 lines
Quoted 4 lines

I'd hate to run into situations where even killing all processes holding some file open leads to a situation where it remains open inside the kernel, thus blocking e.g. unmounting. I already see operators getting very angy ... :o

This is really a different design approach. The idea is to keep an FD active beyond the lifetime of a running process so that we can do panic recovery. Alessio's patchset has similar side effect in some corner cases and this RFC patch makes it a semantic promise. Whether ops like it would really depend on what they want.

The problem is: (most) fd's are bound to some processes - when they're killed, the fd's are closed. Usually you can force a closing files (and thus allow unmount) by checking via lsof which processes still hold open fd to them and kill'em. If we can't do that anymore, we can run into big trouble. There needs to be some clear lifetime control for that.

Let's look at containers: usually the runtime/orchestration sets up a
bunch of mounts before starting the actual workload inside the container. On container shutdown, the processes are killed and then everything's unmounted again (temporary storage, eg. lvm volumes or
btrfs subvols are removed afterwards).

Now, with the persistant fd's, an unprivileged user can block that (either accidentially or on purpose). We cannot allow that to happen.

Apropos containers: this really should be, some how, bound to some namespace (not sure whether mountfs or userns is the right place), so containers cannot interfer with each other.

Quoted 5 lines

I agree but I understand the rationale as well. A normal FUSE read/write uses FUSE daemon creds so the semantics are the same. Otherwise as you outline below, we'd have to go through all the read/write callbacks to make sure none of them is checking process creds.

I've actually looked deeper into this. There indeed are certain places with checks for CAP_SYS_ADMIN, but these are really root-only things where we should think very carefully whether they should work with fds passed to processes under separate users at all. And they also never worked with fd passing via unix socket.

--mtx

Signature · 8 lines
---
Hinweis: unverschlüsselte E-Mails können leicht abgehört und manipuliert
werden ! Für eine vertrauliche Kommunikation senden Sie bitte ihren
GPG/PGP-Schlüssel zu.
---
Enrico Weigelt, metux IT consult
Free software and Linux embedded engineering
[email protected] -- +49-151-27565287