Skip to contentSkip to conversation

fuse: add fusex filesystem

PATCH In review Miklos Szeredi 8 participants 28 messages Apr 29 – Jun 22

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 fuse: add fusex filesystem

This stands for "fuse extended/experimental".

The purpose is to provide a clean base for big features like the FUSE_IOMAP api.

It's also a good way to try new stuff like file handles and compound requests without the risk of breaking something in the large and complex fuse codebase.

Whether these features will be migrated back into the main fuse codebase, or fusex is going to end up as a major version update is still up in the air.

Major differences from regular fuse:

- local filesystem mode only
- only synchronous FUSE_INIT is supported
- only no-open mode
- new requests:
+ FUSE_LOOKUP_ROOT - return nodeid of root
+ FUSE_LOOKUPX - FUSE_LOOKUP without the getattr
+ FUSE_MKOBJX - merged FUSE_MKNOD, MKDIR, SYMLINK and TMPFILE
+ FUSE_SETSTATX - extended version of FUSE_SETATTR

Missing features:

- file handles / export ops
- compound requests
- xattr caching
- fileattr
- fiemap
- ioctl
- copy_file_range
- lazy dir open

Test server can be found at:

https://github.com/szmi/fuse-utils

---

Patch is against

git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git#for-next

 fs/fuse/Makefile          |    2 +-
 fs/fuse/args.h            |    1 +
 fs/fuse/dev.c             |   13 +-
 fs/fuse/dir.c             |    4 +-
 fs/fuse/fuse_i.h          |    8 +
 fs/fuse/fusex.c           | 1751 +++++++++++++++++++++++++++++++++++++
 fs/fuse/fusex.h           |    4 +
 fs/fuse/inode.c           |   16 +-
 include/uapi/linux/fuse.h |   36 +
 9 files changed, 1826 insertions(+), 9 deletions(-)
 create mode 100644 fs/fuse/fusex.c
 create mode 100644 fs/fuse/fusex.h
fs/fuse/Makefile +1 -1
index 245e67852b03..d9963e411b62 100644
@@ -12,7 +12,7 @@ obj-$(CONFIG_VIRTIO_FS) += virtiofs.o
 
 fuse-y := trace.o	# put trace.o first so we see ftrace errors sooner
 fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o req_timeout.o req.o
-fuse-y += poll.o notify.o
+fuse-y += poll.o notify.o fusex.o
 fuse-y += iomode.o
 fuse-$(CONFIG_FUSE_DAX) += dax.o
 fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o backing.o
fs/fuse/args.h +1 -0
index ecfe51a192af..1c1e0a25ea07 100644
@@ -35,6 +35,7 @@ struct fuse_args {
 	bool out_pages:1;
 	bool user_pages:1;
 	bool out_argvar:1;
+	bool out_var_alloc:1;
 	bool page_zeroing:1;
 	bool page_replace:1;
 	bool may_block:1;
fs/fuse/dev.c +11 -2
index 6fe0d8c263df..134572b1a9af 100644
@@ -1847,15 +1847,24 @@ int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args,
 
 	reqsize += fuse_len_args(args->out_numargs, args->out_args);
 
-	if (reqsize < nbytes || (reqsize > nbytes && !args->out_argvar))
+	if (reqsize < nbytes)
 		return -EINVAL;
-	else if (reqsize > nbytes) {
+
+	if (args->out_argvar) {
 		struct fuse_arg *lastarg = &args->out_args[args->out_numargs-1];
 		unsigned diffsize = reqsize - nbytes;
 
 		if (diffsize > lastarg->size)
 			return -EINVAL;
 		lastarg->size -= diffsize;
+
+		if (args->out_var_alloc) {
+			lastarg->value = kvmalloc(lastarg->size, GFP_KERNEL);
+			if (!lastarg->value)
+				return -ENOMEM;
+		}
+	} else if (reqsize > nbytes) {
+		return -EINVAL;
 	}
 	return fuse_copy_args(cs, args->out_numargs, args->out_pages,
 			      args->out_args, args->page_zeroing);
fs/fuse/dir.c +2 -2
index be41c14ef329..cbe0d4b65d49 100644
@@ -542,7 +542,7 @@ int fuse_valid_type(int m)
 		S_ISBLK(m) || S_ISFIFO(m) || S_ISSOCK(m);
 }
 
-static bool fuse_valid_size(u64 size)
+bool fuse_valid_size(u64 size)
 {
 	return size <= LLONG_MAX;
 }
@@ -2485,7 +2485,7 @@ static int fuse_symlink_read_folio(struct file *null, struct folio *folio)
 	return err;
 }
 
-static const struct address_space_operations fuse_symlink_aops = {
+const struct address_space_operations fuse_symlink_aops = {
 	.read_folio	= fuse_symlink_read_folio,
 };
 
fs/fuse/fuse_i.h +8 -0
index 3a7ac74a23ed..fe66281b7554 100644
@@ -69,6 +69,9 @@ extern struct mutex fuse_mutex;
 extern unsigned int max_user_bgreq;
 extern unsigned int max_user_congthresh;
 
+extern struct kmem_cache *fuse_inode_cachep;
+extern const struct address_space_operations fuse_symlink_aops;
+
 struct fuse_forget_link;
 
 /**
@@ -911,6 +914,8 @@ struct inode *fuse_iget(struct super_block *sb, u64 nodeid,
 int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name,
 		     struct fuse_entry_out *outarg, struct inode **inode);
 
+void fuse_umount_begin(struct super_block *sb);
+
 /*
  * Initialize READ or READDIR request
  */
@@ -1102,6 +1107,7 @@ void fuse_ctl_remove_conn(struct fuse_conn *fc);
  * Is file type valid?
  */
 int fuse_valid_type(int m);
+bool fuse_valid_size(u64 size);
 
 bool fuse_invalid_attr(struct fuse_attr *attr);
 
@@ -1204,6 +1210,8 @@ struct posix_acl *fuse_get_acl(struct mnt_idmap *idmap,
 int fuse_set_acl(struct mnt_idmap *, struct dentry *dentry,
 		 struct posix_acl *acl, int type);
 
+void fuse_convert_statfs(struct kstatfs *stbuf, struct fuse_kstatfs *attr);
+
 /* readdir.c */
 int fuse_readdir(struct file *file, struct dir_context *ctx);
 
fs/fuse/fusex.c new file +1751 -0

1,751 lines not on this page · raw message

fs/fuse/fusex.h new file +4 -0
new file mode 100644
index 000000000000..c7b7c6f8b442
@@ -0,0 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+int fusex_init(void);
+void fusex_cleanup(void);
fs/fuse/inode.c +12 -4
index 0897f8e62b4d..2f129f7c168c 100644
@@ -8,6 +8,7 @@
 
 #include "dev.h"
 #include "fuse_i.h"
+#include "fusex.h"
 
 #include <linux/dax.h>
 #include <linux/pagemap.h>
@@ -31,7 +32,7 @@ MODULE_AUTHOR("Miklos Szeredi <[email protected]>");
 MODULE_DESCRIPTION("Filesystem in Userspace");
 MODULE_LICENSE("GPL");
 
-static struct kmem_cache *fuse_inode_cachep;
+struct kmem_cache *fuse_inode_cachep;
 struct list_head fuse_conn_list;
 DEFINE_MUTEX(fuse_mutex);
 
@@ -605,7 +606,7 @@ void fuse_unlock_inode(struct inode *inode, bool locked)
 		mutex_unlock(&get_fuse_inode(inode)->mutex);
 }
 
-static void fuse_umount_begin(struct super_block *sb)
+void fuse_umount_begin(struct super_block *sb)
 {
 	struct fuse_conn *fc = get_fuse_conn_super(sb);
 
@@ -631,7 +632,7 @@ static void fuse_send_destroy(struct fuse_mount *fm)
 	}
 }
 
-static void convert_fuse_statfs(struct kstatfs *stbuf, struct fuse_kstatfs *attr)
+void fuse_convert_statfs(struct kstatfs *stbuf, struct fuse_kstatfs *attr)
 {
 	stbuf->f_type    = FUSE_SUPER_MAGIC;
 	stbuf->f_bsize   = attr->bsize;
@@ -667,7 +668,7 @@ static int fuse_statfs(struct dentry *dentry, struct kstatfs *buf)
 	args.out_args[0].value = &outarg;
 	err = fuse_simple_request(fm, &args);
 	if (!err)
-		convert_fuse_statfs(buf, &outarg.st);
+		fuse_convert_statfs(buf, &outarg.st);
 	return err;
 }
 
@@ -2160,6 +2161,10 @@ static int __init fuse_init(void)
 	if (res)
 		goto err_sysfs_cleanup;
 
+	res = fusex_init();
+	if (res)
+		goto err_ctl_cleanup;
+
 	fuse_dentry_tree_init();
 
 	sanitize_global_limit(&max_user_bgreq);
@@ -2167,6 +2172,8 @@ static int __init fuse_init(void)
 
 	return 0;
 
+err_ctl_cleanup:
+	fuse_ctl_cleanup();
  err_sysfs_cleanup:
 	fuse_sysfs_cleanup();
  err_dev_cleanup:
@@ -2182,6 +2189,7 @@ static void __exit fuse_exit(void)
 	pr_debug("exit\n");
 
 	fuse_dentry_tree_cleanup();
+	fusex_cleanup();
 	fuse_ctl_cleanup();
 	fuse_sysfs_cleanup();
 	fuse_fs_cleanup();
include/uapi/linux/fuse.h +36 -0
index c13e1f9a2f12..f5d90be821ae 100644
@@ -663,6 +663,10 @@ enum fuse_opcode {
 	FUSE_TMPFILE		= 51,
 	FUSE_STATX		= 52,
 	FUSE_COPY_FILE_RANGE_64	= 53,
+	FUSE_LOOKUP_ROOT	= 54,
+	FUSE_LOOKUPX		= 55,
+	FUSE_MKOBJX		= 56,
+	FUSE_SETSTATX		= 58,
 
 	/* CUSE specific operations */
 	CUSE_INIT		= 4096,
@@ -700,6 +704,20 @@ struct fuse_entry_out {
 	struct fuse_attr attr;
 };
 
+/*
+ * entryx flags
+ * FUSE_ENTRYX_NEGATIVE: file does not exist, can cache this result
+ */
+#define FUSE_ENTRYX_NEGATIVE	(1 << 0)
+
+struct fuse_entryx_out {
+	uint64_t	nodeid;
+	uint64_t	entry_valid;
+	uint32_t	entry_valid_nsec;
+	uint32_t	flags;
+	uint64_t	spare;
+};
+
 struct fuse_forget_in {
 	uint64_t	nlookup;
 };
@@ -754,6 +772,17 @@ struct fuse_mknod_in {
 	uint32_t	padding;
 };
 
+enum fuse_mkobjx_flags {
+	FUSE_MKOBJX_TMPFILE = 1 << 0,
+};
+
+struct fuse_mkobjx_in {
+	struct fuse_statx stat;
+	uint32_t	namesize;
+	uint32_t	flags;
+	uint64_t	spare[7];
+};
+
 struct fuse_mkdir_in {
 	uint32_t	mode;
 	uint32_t	umask;
@@ -792,6 +821,13 @@ struct fuse_setattr_in {
 	uint32_t	unused5;
 };
 
+struct fuse_setstatx_in {
+	uint64_t	fh;
+	uint32_t	flags;
+	uint32_t	reserved;
+	struct fuse_statx stat;
+};
+
 struct fuse_open_in {
 	uint32_t	flags;
 	uint32_t	open_flags;	/* FUSE_OPEN_... */
Signature · 1 line
2.53.0
#1 lore raw
Quoted 35 lines · Miklos Szeredi

This stands for "fuse extended/experimental".

The purpose is to provide a clean base for big features like the FUSE_IOMAP api.

It's also a good way to try new stuff like file handles and compound requests without the risk of breaking something in the large and complex fuse codebase.

Whether these features will be migrated back into the main fuse codebase, or fusex is going to end up as a major version update is still up in the air.

Major differences from regular fuse:

- local filesystem mode only
- only synchronous FUSE_INIT is supported
- only no-open mode
- new requests:
+ FUSE_LOOKUP_ROOT - return nodeid of root
+ FUSE_LOOKUPX - FUSE_LOOKUP without the getattr
+ FUSE_MKOBJX - merged FUSE_MKNOD, MKDIR, SYMLINK and TMPFILE
+ FUSE_SETSTATX - extended version of FUSE_SETATTR

Missing features:

- file handles / export ops
- compound requests
- xattr caching
- fileattr
- fiemap
- ioctl
- copy_file_range
- lazy dir open

Quoted 7 lines

Test server can be found at:

https://github.com/szmi/fuse-utils

---

...

Hi Miklos,

I really like the new shiny fusex. It just has one heavy drawback for me. It is not set to support remote file systems.

I have a coupe of questions that I would like to ask here on the list.
What was the reasoning behind this? Complexity, lack of interest from the community/users of fuse?

Where do you see the biggest challenge for this. To me, it doesn't look impossible to add that, but I'm sure I'm missing a lot in this context.

Thanks,
Horst

#2 lore raw
Quoted 11 lines · Horst Birthelmer
Quoted 9 lines · Miklos Szeredi

This stands for "fuse extended/experimental".

The purpose is to provide a clean base for big features like the FUSE_IOMAP api.

It's also a good way to try new stuff like file handles and compound requests without the risk of breaking something in the large and complex fuse codebase.

Quoted 7 lines
Quoted 7 lines

Whether these features will be migrated back into the main fuse codebase, or fusex is going to end up as a major version update is still up in the air.

Major differences from regular fuse:

- local filesystem mode only

What about things like?
FUSE_HANDLE_KILLPRIV | FUSE_HANDLE_KILLPRIV_V2
FUSE_ATOMIC_O_TRUNC

Does local-only mean that all those (and default_permissions) are handled by the kernel?

Quoted 11 lines
Quoted 11 lines

- only synchronous FUSE_INIT is supported
- only no-open mode
- new requests:
+ FUSE_LOOKUP_ROOT - return nodeid of root
+ FUSE_LOOKUPX - FUSE_LOOKUP without the getattr
+ FUSE_MKOBJX - merged FUSE_MKNOD, MKDIR, SYMLINK and TMPFILE
+ FUSE_SETSTATX - extended version of FUSE_SETATTR

Missing features:

- file handles / export ops

Need to sort out this API that grew odd organically over the years:

FUSE_EXPORT_SUPPORT | FUSE_NO_EXPORT_SUPPORT

The problem is that libfuse declares FUSE_EXPORT_SUPPORT on behalf of high level filesystems which do not really provide thse persistent NFS file handle guarantees.

I see no reason for fusex to not support lookup by ".", which is anyway needed for "reconnect", so a fresh FUSEX_NFS_EXPORT opt-in for "re-export to NFS" may make more sense.

Thanks,
Amir.

maintainer
#3 lore raw
Quoted 7 lines · Amir Goldstein
Quoted 5 lines · Horst Birthelmer
Quoted 2 lines · Miklos Szeredi

Major differences from regular fuse:

Quoted 8 lines
Quoted 1 line
Quoted 1 line

- local filesystem mode only

What about things like?
FUSE_HANDLE_KILLPRIV | FUSE_HANDLE_KILLPRIV_V2
FUSE_ATOMIC_O_TRUNC

Does local-only mean that all those (and default_permissions) are handled by the kernel?

Yes.

Quoted 3 lines

I see no reason for fusex to not support lookup by ".", which is anyway needed for "reconnect", so a fresh FUSEX_NFS_EXPORT opt-in for "re-export to NFS" may make more sense.

Why add reconnect? If server supports persistent file handles, and all req's get the file handle, then no state (nodeid) needs to be stored by the server.

Thanks,
Miklos

#4 lore raw
Quoted 9 lines · Miklos Szeredi
Quoted 4 lines

I see no reason for fusex to not support lookup by ".", which is anyway needed for "reconnect", so a fresh FUSEX_NFS_EXPORT opt-in for "re-export to NFS" may make more sense.

Why add reconnect? If server supports persistent file handles, and all req's get the file handle, then no state (nodeid) needs to be stored by the server.

Yes, we are saying the same thing.
What I am saying is that legacy fuse has FUSE_EXPORT_SUPPORT which means server supports lookup of "." which libfuse always enabled and then we added FUSE_NO_EXPORT_SUPPORT to opt-out of nfs export.

I am saying that in FUSEX, lookup "." support is implied. And that nfs export should always be opt-in (FUSEX_NFS_EXPORT).

Thanks,
Amir.

#5 lore raw
Quoted 11 lines · Amir Goldstein
Quoted 9 lines · Miklos Szeredi
Quoted 4 lines

I see no reason for fusex to not support lookup by ".", which is anyway needed for "reconnect", so a fresh FUSEX_NFS_EXPORT opt-in for "re-export to NFS" may make more sense.

Why add reconnect? If server supports persistent file handles, and all req's get the file handle, then no state (nodeid) needs to be stored by the server.

Quoted 7 lines

Yes, we are saying the same thing.
What I am saying is that legacy fuse has FUSE_EXPORT_SUPPORT which means server supports lookup of "." which libfuse always enabled and then we added FUSE_NO_EXPORT_SUPPORT to opt-out of nfs export.

I am saying that in FUSEX, lookup "." support is implied. And that nfs export should always be opt-in (FUSEX_NFS_EXPORT).

I'm confused here, why do we need lookup of "." without NFS export? And which file system actually supports lookup of "."?
Shouldn't all that be scratched in favor of LOOKUP_HANDLE?

Thanks,
Bernd

#6 lore raw
Quoted 20 lines · Bernd Schubert
Quoted 16 lines · Amir Goldstein
Quoted 9 lines · Miklos Szeredi
Quoted 4 lines

I see no reason for fusex to not support lookup by ".", which is anyway needed for "reconnect", so a fresh FUSEX_NFS_EXPORT opt-in for "re-export to NFS" may make more sense.

Why add reconnect? If server supports persistent file handles, and all req's get the file handle, then no state (nodeid) needs to be stored by the server.

Yes, we are saying the same thing.
What I am saying is that legacy fuse has FUSE_EXPORT_SUPPORT which means server supports lookup of "." which libfuse always enabled and then we added FUSE_NO_EXPORT_SUPPORT to opt-out of nfs export.

Quoted 6 lines
Quoted 2 lines

I am saying that in FUSEX, lookup "." support is implied. And that nfs export should always be opt-in (FUSEX_NFS_EXPORT).

I'm confused here, why do we need lookup of "." without NFS export? And which file system actually supports lookup of "."?
Shouldn't all that be scratched in favor of LOOKUP_HANDLE?

Yes. This is exactly what I meant by lookup "." support is implied.

AFAICT, libfuse sets CAP_EXPORT_SUPPORT unconditionally:

        fuse_set_feature_flag(conn, FUSE_CAP_EXPORT_SUPPORT);

ALL of the high level fuse fs are allowed to be exported by NFS, which is often a very bad idea.

Thanks,
Amir.

#7 lore raw
Quoted 28 lines · Amir Goldstein
Quoted 20 lines · Bernd Schubert
Quoted 16 lines · Amir Goldstein
Quoted 9 lines · Miklos Szeredi
Quoted 4 lines

I see no reason for fusex to not support lookup by ".", which is anyway needed for "reconnect", so a fresh FUSEX_NFS_EXPORT opt-in for "re-export to NFS" may make more sense.

Why add reconnect? If server supports persistent file handles, and all req's get the file handle, then no state (nodeid) needs to be stored by the server.

Yes, we are saying the same thing.
What I am saying is that legacy fuse has FUSE_EXPORT_SUPPORT which means server supports lookup of "." which libfuse always enabled and then we added FUSE_NO_EXPORT_SUPPORT to opt-out of nfs export.

Quoted 6 lines
Quoted 2 lines

I am saying that in FUSEX, lookup "." support is implied. And that nfs export should always be opt-in (FUSEX_NFS_EXPORT).

I'm confused here, why do we need lookup of "." without NFS export? And which file system actually supports lookup of "."?
Shouldn't all that be scratched in favor of LOOKUP_HANDLE?

Quoted 8 lines

Yes. This is exactly what I meant by lookup "." support is implied.

AFAICT, libfuse sets CAP_EXPORT_SUPPORT unconditionally:

        fuse_set_feature_flag(conn, FUSE_CAP_EXPORT_SUPPORT);

ALL of the high level fuse fs are allowed to be exported by NFS, which is often a very bad idea.

I think that predates me and is in the came category as FUSE_CAP_AUTO_INVAL_DATA. Initially I thought it gets really complicated and I need to add another ABI/API compat layer to encode into struct fuse_session which FUSE_USE_VERSION the application uses, but then in the discussion with Darrick we got the idea to put it into the existing padding

fuse_session_new_fn(struct fuse_args *args, const struct fuse_lowlevel_ops *op,
size_t op_size, void *userdata)
{
struct libfuse_version version = {
.major = FUSE_MAJOR_VERSION, .minor = FUSE_MINOR_VERSION, .hotfix = FUSE_HOTFIX_VERSION, .padding = 0
};

return fuse_session_new_versioned(args, op, op_size, &version,
userdata);
}

So idea and suggestion Miklos for FUSE_CAP_AUTO_INVAL_DATA was to chage the default based on FUSE_USE_VERSION. We can do the same for FUSE_CAP_EXPORT_SUPPORT.

The initial patch to add in that version into libfuse as another function argument got really complex in the high level layer and then QA team at DDN figure out that disabling FUSE_CAP_AUTO_INVAL_DATA actually triggers another data corruption related xftest error (reproducible with passthrough_hp) and I didn't have time to follow up that chain of issues
- got stalled for now, but I need to resume it (unless someone else wants to work on it).

Thanks,
Bernd

maintainer
#8 lore raw
In reply to Horst Birthelmer
Quoted 2 lines · Horst Birthelmer

Where do you see the biggest challenge for this. To me, it doesn't look impossible to add that, but I'm sure I'm missing a lot in this context.

The biggest challenge I see is API design. I'd really like to have a simple and consistent API for this.

First task: need to document the current interface, which is long overdue. Here's a start:

https://docs.google.com/document/d/1SInG6nc5dF-db3WtCqFOnRDCY1jRl5JQ0Toiryn2PRc/edit?usp=sharing

This needs refinement, i.e. try to document by cache type, how is the cache used, invalidated, etc.

Thanks,
Miklos

#9 lore raw
Quoted 7 lines · Miklos Szeredi
Quoted 2 lines · Horst Birthelmer

Where do you see the biggest challenge for this. To me, it doesn't look impossible to add that, but I'm sure I'm missing a lot in this context.

The biggest challenge I see is API design. I'd really like to have a simple and consistent API for this.

OK, that is a valid goal, I guess.

I have tried to come up with a fusex 'module' that does all operations over io-uring since it is a lot easier to just support one way of transport.

Then I have written a very primitive passthrough to test the user space interface and I can get it to work with some small changes to the interface mostly due to the small header space on the ring.

I have done this to check how fast I can have something workable with the new interface since I'm supposed to add compound commands.

Do you think it is worth pursuing further?

Quoted 11 lines

First task: need to document the current interface, which is long overdue. Here's a start:

https://docs.google.com/document/d/1SInG6nc5dF-db3WtCqFOnRDCY1jRl5JQ0Toiryn2PRc/edit?usp=sharing

This needs refinement, i.e. try to document by cache type, how is the cache used, invalidated, etc.

Thanks,
Miklos

#10 lore raw
In reply to Miklos Szeredi
Quoted 190 lines · Miklos Szeredi

This stands for "fuse extended/experimental".

The purpose is to provide a clean base for big features like the FUSE_IOMAP api.

It's also a good way to try new stuff like file handles and compound requests without the risk of breaking something in the large and complex fuse codebase.

Whether these features will be migrated back into the main fuse codebase, or fusex is going to end up as a major version update is still up in the air.

Major differences from regular fuse:

- local filesystem mode only
- only synchronous FUSE_INIT is supported
- only no-open mode
- new requests:
+ FUSE_LOOKUP_ROOT - return nodeid of root
+ FUSE_LOOKUPX - FUSE_LOOKUP without the getattr
+ FUSE_MKOBJX - merged FUSE_MKNOD, MKDIR, SYMLINK and TMPFILE
+ FUSE_SETSTATX - extended version of FUSE_SETATTR

Missing features:

- file handles / export ops
- compound requests
- xattr caching
- fileattr
- fiemap
- ioctl
- copy_file_range
- lazy dir open

Test server can be found at:

https://github.com/szmi/fuse-utils

---

Patch is against

git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git#for-next

 fs/fuse/Makefile          |    2 +-
 fs/fuse/args.h            |    1 +
 fs/fuse/dev.c             |   13 +-
 fs/fuse/dir.c             |    4 +-
 fs/fuse/fuse_i.h          |    8 +
 fs/fuse/fusex.c           | 1751 +++++++++++++++++++++++++++++++++++++
 fs/fuse/fusex.h           |    4 +
 fs/fuse/inode.c           |   16 +-
 include/uapi/linux/fuse.h |   36 +
 9 files changed, 1826 insertions(+), 9 deletions(-)
 create mode 100644 fs/fuse/fusex.c
 create mode 100644 fs/fuse/fusex.h
fs/fuse/Makefile +1 -1
index 245e67852b03..d9963e411b62 100644
@@ -12,7 +12,7 @@ obj-$(CONFIG_VIRTIO_FS) += virtiofs.o
 
 fuse-y := trace.o	# put trace.o first so we see ftrace errors sooner
 fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o req_timeout.o req.o
-fuse-y += poll.o notify.o
+fuse-y += poll.o notify.o fusex.o
 fuse-y += iomode.o
 fuse-$(CONFIG_FUSE_DAX) += dax.o
 fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o backing.o
fs/fuse/args.h +1 -0
index ecfe51a192af..1c1e0a25ea07 100644
@@ -35,6 +35,7 @@ struct fuse_args {
 	bool out_pages:1;
 	bool user_pages:1;
 	bool out_argvar:1;
+	bool out_var_alloc:1;
 	bool page_zeroing:1;
 	bool page_replace:1;
 	bool may_block:1;
fs/fuse/dev.c +11 -2
index 6fe0d8c263df..134572b1a9af 100644
@@ -1847,15 +1847,24 @@ int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args,
 
 	reqsize += fuse_len_args(args->out_numargs, args->out_args);
 
-	if (reqsize < nbytes || (reqsize > nbytes && !args->out_argvar))
+	if (reqsize < nbytes)
 		return -EINVAL;
-	else if (reqsize > nbytes) {
+
+	if (args->out_argvar) {
 		struct fuse_arg *lastarg = &args->out_args[args->out_numargs-1];
 		unsigned diffsize = reqsize - nbytes;
 
 		if (diffsize > lastarg->size)
 			return -EINVAL;
 		lastarg->size -= diffsize;
+
+		if (args->out_var_alloc) {
+			lastarg->value = kvmalloc(lastarg->size, GFP_KERNEL);
+			if (!lastarg->value)
+				return -ENOMEM;
+		}
+	} else if (reqsize > nbytes) {
+		return -EINVAL;
 	}
 	return fuse_copy_args(cs, args->out_numargs, args->out_pages,
 			      args->out_args, args->page_zeroing);
fs/fuse/dir.c +2 -2
index be41c14ef329..cbe0d4b65d49 100644
@@ -542,7 +542,7 @@ int fuse_valid_type(int m)
 		S_ISBLK(m) || S_ISFIFO(m) || S_ISSOCK(m);
 }
 
-static bool fuse_valid_size(u64 size)
+bool fuse_valid_size(u64 size)
 {
 	return size <= LLONG_MAX;
 }
@@ -2485,7 +2485,7 @@ static int fuse_symlink_read_folio(struct file *null, struct folio *folio)
 	return err;
 }
 
-static const struct address_space_operations fuse_symlink_aops = {
+const struct address_space_operations fuse_symlink_aops = {
 	.read_folio	= fuse_symlink_read_folio,
 };
 
fs/fuse/fuse_i.h +8 -0
index 3a7ac74a23ed..fe66281b7554 100644
@@ -69,6 +69,9 @@ extern struct mutex fuse_mutex;
 extern unsigned int max_user_bgreq;
 extern unsigned int max_user_congthresh;
 
+extern struct kmem_cache *fuse_inode_cachep;
+extern const struct address_space_operations fuse_symlink_aops;
+
 struct fuse_forget_link;
 
 /**
@@ -911,6 +914,8 @@ struct inode *fuse_iget(struct super_block *sb, u64 nodeid,
 int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name,
 		     struct fuse_entry_out *outarg, struct inode **inode);
 
+void fuse_umount_begin(struct super_block *sb);
+
 /*
  * Initialize READ or READDIR request
  */
@@ -1102,6 +1107,7 @@ void fuse_ctl_remove_conn(struct fuse_conn *fc);
  * Is file type valid?
  */
 int fuse_valid_type(int m);
+bool fuse_valid_size(u64 size);
 
 bool fuse_invalid_attr(struct fuse_attr *attr);
 
@@ -1204,6 +1210,8 @@ struct posix_acl *fuse_get_acl(struct mnt_idmap *idmap,
 int fuse_set_acl(struct mnt_idmap *, struct dentry *dentry,
 		 struct posix_acl *acl, int type);
 
+void fuse_convert_statfs(struct kstatfs *stbuf, struct fuse_kstatfs *attr);
+
 /* readdir.c */
 int fuse_readdir(struct file *file, struct dir_context *ctx);
 
fs/fuse/fusex.c new file +7 -0
new file mode 100644
index 000000000000..98e239e7e00e
@@ -0,0 +1,1751 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include "fusex.h"
+#include "dev.h"
+#include "fuse_i.h"
+
+#include <linux/fs_context.h>

fs/fuse/fusex.c line 17

Quoted 10 lines
fs/fuse/fusex.c new file +10 -0
+#include <linux/miscdevice.h>
+#include <linux/xxhash.h>
+#include <linux/pagemap.h>
+#include <linux/exportfs.h>
+#include <linux/iversion.h>
+#include <linux/posix_acl_xattr.h>
+#include <linux/statfs.h>
+#include <linux/falloc.h>
+#include <linux/fs_parser.h>
+

I think you're missing
#include <uapi/linux/magic.h>
here for FUSE_SUPER_MAGIC

maintainer
#11 lore raw
Quoted 3 lines · Horst Birthelmer

I think you're missing
#include <uapi/linux/magic.h>
here for FUSE_SUPER_MAGIC

Okay.

Thanks,
Miklos

#12 lore raw
In reply to Miklos Szeredi
Quoted 35 lines · Miklos Szeredi

This stands for "fuse extended/experimental".

The purpose is to provide a clean base for big features like the FUSE_IOMAP api.

It's also a good way to try new stuff like file handles and compound requests without the risk of breaking something in the large and complex fuse codebase.

Whether these features will be migrated back into the main fuse codebase, or fusex is going to end up as a major version update is still up in the air.

Major differences from regular fuse:

- local filesystem mode only
- only synchronous FUSE_INIT is supported
- only no-open mode
- new requests:
+ FUSE_LOOKUP_ROOT - return nodeid of root
+ FUSE_LOOKUPX - FUSE_LOOKUP without the getattr
+ FUSE_MKOBJX - merged FUSE_MKNOD, MKDIR, SYMLINK and TMPFILE
+ FUSE_SETSTATX - extended version of FUSE_SETATTR

Missing features:

- file handles / export ops
- compound requests
- xattr caching
- fileattr
- fiemap
- ioctl
- copy_file_range
- lazy dir open

Quoted 7 lines

Test server can be found at:

https://github.com/szmi/fuse-utils

---

Hi Miklos,

I have patched passthrough_hp to use fusex and have noticed, that I had to do the following to make it work:

fs/fuse/fusex.c +2 -0
index 98e239e7e00e..87fcfc16645a 100644
@@ -14,6 +14,7 @@
 #include <linux/statfs.h>
 #include <linux/falloc.h>
 #include <linux/fs_parser.h>
+#include <uapi/linux/magic.h>
 
 static void fusex_init_inode(struct inode *inode);
 
@@ -1337,6 +1338,7 @@ static int fusex_send_opendir(struct fuse_file *ff, struct inode *inode)
        if (!err) {
                ff->fh = outarg.fh;
                ff->open_flags = FOPEN_CACHE_DIR;
+               ff->nodeid = get_node_id(inode);
        }
        return err;
 }

---

After that I could actually mount and do simple meta operations. The nodeid is used for listing.

Thanks,
Horst

maintainer
#13 lore raw

Review static int fusex_send_opendir(struct fuse_file *ff, struct inode *inode) · line 1341

Quoted 5 lines · Horst Birthelmer
(quoted diff) +1 -0
@@ -1337,6 +1338,7 @@ static int fusex_send_opendir(struct fuse_file *ff, struct inode *inode)
        if (!err) {
                ff->fh = outarg.fh;
                ff->open_flags = FOPEN_CACHE_DIR;
+               ff->nodeid = get_node_id(inode);

How about this (untested):

fs/fuse/file.c +0 -0
index 3bdab8d03373..1b5a262147ef 100644
@@ -594,7 +594,7 @@ void fuse_read_args_fill(struct fuse_io_args *ia,

struct file *file, loff_t pos,

        ia->read.in.size = count;
        ia->read.in.flags = file->f_flags;
        args->opcode = opcode;
-       args->nodeid = ff->nodeid;
+       args->nodeid = get_node_id(file_inode(file));
        args->in_numargs = 1;
        args->in_args[0].size = sizeof(ia->read.in);
        args->in_args[0].value = &ia->read.in;
fs/fuse/fusex.c +0 -0
index c4bc03e57ed7..43c03d7f73ab 100644
@@ -1379,6 +1380,7 @@ static int fusex_dir_release(struct inode

*inode, struct file *file)

        ra->inarg.fh = ff->fh;

        ra->args.opcode = FUSE_RELEASEDIR;
+       ra->args.nodeid = get_node_id(inode);
        ra->args.force = true;
        ra->args.nocreds = true;
        ra->args.end = fusex_release_end;

Thanks,
Miklos

fs/fuse/file.c +1 -1
index 3bdab8d03373..1b5a262147ef 100644
@@ -594,7 +594,7 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
        ia->read.in.size = count;
        ia->read.in.flags = file->f_flags;
        args->opcode = opcode;
-       args->nodeid = ff->nodeid;
+       args->nodeid = get_node_id(file_inode(file));
        args->in_numargs = 1;
        args->in_args[0].size = sizeof(ia->read.in);
        args->in_args[0].value = &ia->read.in;
fs/fuse/fusex.c +1 -0
index c4bc03e57ed7..43c03d7f73ab 100644
@@ -1379,6 +1380,7 @@ static int fusex_dir_release(struct inode *inode, struct file *file)
        ra->inarg.fh = ff->fh;
 
        ra->args.opcode = FUSE_RELEASEDIR;
+       ra->args.nodeid = get_node_id(inode);
        ra->args.force = true;
        ra->args.nocreds = true;
        ra->args.end = fusex_release_end;
#14 lore raw
In reply to Miklos Szeredi
Quoted 6 lines · Miklos Szeredi

This stands for "fuse extended/experimental".

The purpose is to provide a clean base for big features like the FUSE_IOMAP api.

Quoted 7 lines

It's also a good way to try new stuff like file handles and compound requests without the risk of breaking something in the large and complex fuse codebase.

Whether these features will be migrated back into the main fuse codebase, or fusex is going to end up as a major version update is still up in the air.

Could you explain a bit what the main motivation behind fusex is? Is it mainly to clean up interfaces that are now clunky/stale? Or is it more driven by limitations in the current codebase that make newer features hard to add? From the paragraphs above, it kind of seems like the latter?

If it's mostly to clean up interfaces that are now clunky/stale, I wonder if now is the best time given that, as Amir phrased it in [1], "FUSE is experiencing a renaissance of new features and protocol enhancements", and I'm not sure if we know yet if these new interfaces will also have some things we'll have wished in hindsight we had done differently, especially since some of these new features are still actively evolving and gaining new capabilities (including io-uring and passthrough). Would it make sense to let these new features and protocol enhancements evolve and stabilize first before baking them into fusex?

There's a good chance I'm missing something here, but if fusex is mostly to make new features easier to add, I'm not sure I see a lot of difference between doing it in fuse vs fusex when fusex is expanded to have feature-parity with legacy fuse. Is there something in particular that fusex makes easier?

I wonder if libfuse could serve as a bridge for the gap between legacy fuse and fusex, eg libfuse filling any missing gaps / translating between legacy fuse protocols and fusex protocols such that any fuse server written for legacy fuse could run using fusex, which would remove the need to have to do any "backporting" to legacy fuse for new features that are added to fusex and could accelerate the deprecation timeline of legacy fuse.

Quoted 5 lines

Major differences from regular fuse:

- local filesystem mode only
- only synchronous FUSE_INIT is supported

Some other things that might be nice:
* handle_killpriv / handle_killpriv2 - sgid/suid + file capabilities clearing will always be handled by the server (default implementation in libfuse)
* in fuse io-uring: headers and payload passed together as one contiguous buffer during registration for pinning, deprecate passing separate buffers per SQE
* in passthrough: backing_fd passed at lookup/creation time only (not at open time), expanded struct fuse_backing_map

Quoted 4 lines

- only no-open mode
- new requests:
+ FUSE_LOOKUP_ROOT - return nodeid of root
+ FUSE_LOOKUPX - FUSE_LOOKUP without the getattr

Could you explain why it's preferred to have lookup disentangled from attributes, eg why compounding lookup + attribute fetching is preferred over embedding the statx directly inside the lookup request without compounding? it was brought up in Luis's lookup handles patchset [2] but it's still a bit unclear to me why compounding the two is better. I think every operation/request that returns a struct fuse_entry_out uses the attributes immediately after through fuse_iget(..., &outarg->attr, ...) or fuse_change_attributes(..., &outarg.attr, ...), so the attributes seemed pretty tightly intercoupled with lookup from what I could see.

Thanks,
Joanne

[1] https://lore.kernel.org/linux-fsdevel/CAOQ4uxjRgQGxWYWbRbjVZy19oA5qKZSd9eANKyD8yCoBNGPNNw@mail.gmail.com/
[2] https://lore.kernel.org/linux-fsdevel/[email protected]/T/#t

Quoted 38 lines

+ FUSE_MKOBJX - merged FUSE_MKNOD, MKDIR, SYMLINK and TMPFILE
+ FUSE_SETSTATX - extended version of FUSE_SETATTR

Missing features:

- file handles / export ops
- compound requests
- xattr caching
- fileattr
- fiemap
- ioctl
- copy_file_range
- lazy dir open

Test server can be found at:

https://github.com/szmi/fuse-utils

---

Patch is against

git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git#for-next

 fs/fuse/Makefile          |    2 +-
 fs/fuse/args.h            |    1 +
 fs/fuse/dev.c             |   13 +-
 fs/fuse/dir.c             |    4 +-
 fs/fuse/fuse_i.h          |    8 +
 fs/fuse/fusex.c           | 1751 +++++++++++++++++++++++++++++++++++++
 fs/fuse/fusex.h           |    4 +
 fs/fuse/inode.c           |   16 +-
 include/uapi/linux/fuse.h |   36 +
 9 files changed, 1826 insertions(+), 9 deletions(-)
 create mode 100644 fs/fuse/fusex.c
 create mode 100644 fs/fuse/fusex.h
maintainer
#15 lore raw
Quoted 5 lines · Joanne Koong

Could you explain a bit what the main motivation behind fusex is? Is it mainly to clean up interfaces that are now clunky/stale? Or is it more driven by limitations in the current codebase that make newer features hard to add? From the paragraphs above, it kind of seems like the latter?

I think the answer is: both.

Adding new features also hard because the old interface is convoluted and inconsistent.

Quoted 10 lines

If it's mostly to clean up interfaces that are now clunky/stale, I wonder if now is the best time given that, as Amir phrased it in [1], "FUSE is experiencing a renaissance of new features and protocol enhancements", and I'm not sure if we know yet if these new interfaces will also have some things we'll have wished in hindsight we had done differently, especially since some of these new features are still actively evolving and gaining new capabilities (including io-uring and passthrough). Would it make sense to let these new features and protocol enhancements evolve and stabilize first before baking them into fusex?

Excellent point.

Currently fusex concerns only the filesystem layer, and is agnostic to the transport layer (except maybe the SYNC_INIT thing). That doesn't mean that we shouldn't have the same API cleanup in the transport layer as well. For example, I'd love to get rid of splice on /dev/fuse, which I feel is more burden than blessing. And I guess fuse-uring could also do with an API cleanup.

As to when and how this should be done? I guess that's up to you and Bernd to decide.

But I feel that we do need to have more high level discussion of APIs, if we want to avoid repeating past mistakes.

Quoted 7 lines

I wonder if libfuse could serve as a bridge for the gap between legacy fuse and fusex, eg libfuse filling any missing gaps / translating between legacy fuse protocols and fusex protocols such that any fuse server written for legacy fuse could run using fusex, which would remove the need to have to do any "backporting" to legacy fuse for new features that are added to fusex and could accelerate the deprecation timeline of legacy fuse.

Definitely. However libfuse does export a lot of details from the kernel API (e.g. INIT flags) that would no longer work with fusex and not really feasible to emulate with libfuse. So some porting would be inevitable.

Quoted 4 lines

Some other things that might be nice:
* handle_killpriv / handle_killpriv2 - sgid/suid + file capabilities clearing will always be handled by the server (default implementation in libfuse)

Which is in total contrast to what fusex now does (all that handled by the VFS). Need to resolve this sanely somehow.

Quoted 3 lines

* in fuse io-uring: headers and payload passed together as one contiguous buffer during registration for pinning, deprecate passing separate buffers per SQE

Okay.

Quoted 2 lines

* in passthrough: backing_fd passed at lookup/creation time only (not at open time), expanded struct fuse_backing_map

Yes, this needs discussion. Amir may have additional info on this.

Quoted 6 lines
Quoted 5 lines

- only no-open mode
- new requests:
+ FUSE_LOOKUP_ROOT - return nodeid of root
+ FUSE_LOOKUPX - FUSE_LOOKUP without the getattr

Quoted 9 lines

Could you explain why it's preferred to have lookup disentangled from attributes, eg why compounding lookup + attribute fetching is preferred over embedding the statx directly inside the lookup request without compounding? it was brought up in Luis's lookup handles patchset [2] but it's still a bit unclear to me why compounding the two is better. I think every operation/request that returns a struct fuse_entry_out uses the attributes immediately after through fuse_iget(..., &outarg->attr, ...) or fuse_change_attributes(..., &outarg.attr, ...),

That's one argument for decoupling: why include the same fields (attributes, validity) in several ops (LOOKUP_ROOT, LOOKUPX, MKOBJX) if we already have a separate one to do that? It simplifies the protocol. If we started with decoupled GETATTR, then we wouldn't be taking about adding these X versions, because it would be a simple GETATTR -> STATX conversion.

There's also an interesting side effect of coupling STATX with LOOKUP, which is inherently racy:

Thread A:
send LOOKUP + STATX for /fuse/dir1/filea
[server performs lookup and statx, sends reply]

Thread B:
send SETATTR for /fuse/dir2/fileb, which is hard linked to /fuse/dir1/filea evict inode for /fuse/dir2/fileb

Thread A:
looks up inode for /fuse/dir1/filea in cache, doesn't find any creates inode, fills stale attributes

This issue is currently solved with fc->evict_ctr in regular fuse, but works fine without any additional checks in fusex because of the decoupled ops.

Of course, it is probably desirable to compound LOOKUP with STATX despite this, but it gives the implementation more leeway (e.g. on a different OS).

To conclude: separating out STATX from LOOKUPX, etc. makes the protocol:

a) simpler
b) more flexible

Thanks,
Miklos

#16 lore raw
Quoted 6 lines · Miklos Szeredi
Quoted 3 lines

* in passthrough: backing_fd passed at lookup/creation time only (not at open time), expanded struct fuse_backing_map

Yes, this needs discussion. Amir may have additional info on this.

Having the two different lifecycles (open-to-close, lookup-to-forget) is by design.

This allows the server to potentially move files from passthrough mode to caching mode and vice versa, as long as all fds are closed.

Adding inode passthrough ops does not play well with this mode this is why it requires the lookup-to-forget lifecycle, but I see no reason kill the open-to-close lifecycle as it seems useful for servers that mostly care about I/O passthrough.

w.r.t expanded struct fuse_backing_map, at the moment it does not sound right to me to map a backing id X with more than a single backing file or device.

I think that the maps of fuse_file to multiple fuse_backing is better handled by another layer of mapping (sort of a backing_mapper) which is what famfs and iomap need, but it is just my initial thoughts.

Joanne, if you are a concrete use case for that, let's try to see
how a design to this use case may look like.

Thanks,
Amir.

#17 lore raw
Quoted 6 lines · Amir Goldstein
Quoted 4 lines · Miklos Szeredi
Quoted 3 lines

* in passthrough: backing_fd passed at lookup/creation time only (not at open time), expanded struct fuse_backing_map

Quoted 8 lines
Quoted 2 lines

Yes, this needs discussion. Amir may have additional info on this.

Having the two different lifecycles (open-to-close, lookup-to-forget) is by design.

This allows the server to potentially move files from passthrough mode to caching mode and vice versa, as long as all fds are closed.

I'm unsure how useful mode switching is between caching and passthrough because the server has no control over when the client application closes all its fds (eg it can only react but not trigger it). For my internal use case of switching on passthrough, I was envisioning the server handling everything through direct io until passthrough is possible (eg the remote file has been fully downloaded over the network) so that passthrough can be switched on even with the application having open fds.

Quoted 9 lines

Adding inode passthrough ops does not play well with this mode this is why it requires the lookup-to-forget lifecycle, but I see no reason kill the open-to-close lifecycle as it seems useful for servers that mostly care about I/O passthrough.

w.r.t expanded struct fuse_backing_map, at the moment it does not sound right to me to map a backing id X with more than a single backing file or device.

My thinking with the struct fuse_backing_map expansion was that future ioctls for managing backing files (eg attaching a backing file to a live inode or swapping one backing file for another) might benefit from a shared struct instead of needing to add new structs for each type.

Quoted 4 lines

I think that the maps of fuse_file to multiple fuse_backing is better handled by another layer of mapping (sort of a backing_mapper) which is what famfs and iomap need, but it is just my initial thoughts.

I like this idea a lot, it makes sense to me.

Quoted 3 lines

Joanne, if you are a concrete use case for that, let's try to see
how a design to this use case may look like.

The internal team I'm working with is planning to start working on their server in H2. They will run some benchmarks to see if current passthrough and/or io-uring+zero-copy might get them the performance gains they need, or whether they will still need the set of advanced passthrough features they requested. I'll keep you posted once there's performance data and if there is still a need for the more advanced passthrough features (file mapping to multiple backing files, backing files dynamically changing, enabling passthrough on a live inode), I'll draft up a design doc to get your thoughts on. In the meantime, I'm planning to send out v2 of the inode metadata passthrough patchset sometime this week or next with your feedback from v1.

Thanks,
Joanne

Quoted 3 lines

Thanks,
Amir.

#18 lore raw
Quoted 14 lines · Joanne Koong
Quoted 12 lines · Amir Goldstein
Quoted 6 lines · Miklos Szeredi
Quoted 3 lines

* in passthrough: backing_fd passed at lookup/creation time only (not at open time), expanded struct fuse_backing_map

Yes, this needs discussion. Amir may have additional info on this.

Having the two different lifecycles (open-to-close, lookup-to-forget) is by design.

Quoted 7 lines
Quoted 2 lines

This allows the server to potentially move files from passthrough mode to caching mode and vice versa, as long as all fds are closed.

I'm unsure how useful mode switching is between caching and passthrough because the server has no control over when the client application closes all its fds (eg it can only react but not trigger it).

This is debatable.
My passthrough fs server coordinates state changes with HSM using flock on open files.

Previous implementations used to allow using cached mode until the file content is fully downloaded, which could be beneficial when streaming large media files.

Current implementation uses the DIO or PASSTHROUGH modes. I have not yet collected enough real world stats to know if this is a problem for large media files streaming, so would prefer to keep the options open for now and I see no reason to deprecate the open-to-close lifecycle. I don't think is adds that much complexity to the code (?).

Quoted 6 lines

For my internal use case of switching on passthrough, I was envisioning the server handling everything through direct io until passthrough is possible (eg the remote file has been fully downloaded over the network) so that passthrough can be switched on even with the application having open fds.

Indeed, being able to switch to passthrough for new fds is a big win, which is why I also changed to this method in the current implementation.

Quoted 27 lines
Quoted 9 lines

Adding inode passthrough ops does not play well with this mode this is why it requires the lookup-to-forget lifecycle, but I see no reason kill the open-to-close lifecycle as it seems useful for servers that mostly care about I/O passthrough.

w.r.t expanded struct fuse_backing_map, at the moment it does not sound right to me to map a backing id X with more than a single backing file or device.

My thinking with the struct fuse_backing_map expansion was that future ioctls for managing backing files (eg attaching a backing file to a live inode or swapping one backing file for another) might benefit from a shared struct instead of needing to add new structs for each type.

Quoted 4 lines

I think that the maps of fuse_file to multiple fuse_backing is better handled by another layer of mapping (sort of a backing_mapper) which is what famfs and iomap need, but it is just my initial thoughts.

I like this idea a lot, it makes sense to me.

Quoted 3 lines

Joanne, if you are a concrete use case for that, let's try to see
how a design to this use case may look like.

Quoted 9 lines

The internal team I'm working with is planning to start working on their server in H2. They will run some benchmarks to see if current passthrough and/or io-uring+zero-copy might get them the performance gains they need, or whether they will still need the set of advanced passthrough features they requested. I'll keep you posted once there's performance data and if there is still a need for the more advanced passthrough features (file mapping to multiple backing files, backing files dynamically changing, enabling passthrough on a live inode), I'll draft up a design doc to get your thoughts on.

Sounds good.
FYI, some of those issues we try to solve at application layer. Our HSM application always tries to keep the inode itself across state changes.
For example, by using clone_file_range() to fill in content. With xfs, we could also try to use atomic swap extents, if file content can change, instead of changing inode, but we did not try this yet.

Quoted 4 lines

In the meantime,
I'm planning to send out v2 of the inode metadata passthrough patchset sometime this week or next with your feedback from v1.

Cool. Let's get this rolling.
I need to get my test setup back in shape.
There were a bunch of fstest failing last time I tested passthrough, mostly related to metadata cache coherency.
I was hoping that GETATTR passthrough could solve some of them. If you have seen any improvements (or regressions) in fstests please share this info with the patch set.

Thanks,
Amir.

#19 lore raw
Quoted 86 lines · Amir Goldstein
Quoted 14 lines · Joanne Koong
Quoted 12 lines · Amir Goldstein
Quoted 6 lines · Miklos Szeredi
Quoted 3 lines

* in passthrough: backing_fd passed at lookup/creation time only (not at open time), expanded struct fuse_backing_map

Yes, this needs discussion. Amir may have additional info on this.

Having the two different lifecycles (open-to-close, lookup-to-forget) is by design.

Quoted 7 lines
Quoted 2 lines

This allows the server to potentially move files from passthrough mode to caching mode and vice versa, as long as all fds are closed.

I'm unsure how useful mode switching is between caching and passthrough because the server has no control over when the client application closes all its fds (eg it can only react but not trigger it).

This is debatable.
My passthrough fs server coordinates state changes with HSM using flock on open files.

Previous implementations used to allow using cached mode until the file content is fully downloaded, which could be beneficial when streaming large media files.

Current implementation uses the DIO or PASSTHROUGH modes. I have not yet collected enough real world stats to know if this is a problem for large media files streaming, so would prefer to keep the options open for now and I see no reason to deprecate the open-to-close lifecycle. I don't think is adds that much complexity to the code (?).

Quoted 6 lines

For my internal use case of switching on passthrough, I was envisioning the server handling everything through direct io until passthrough is possible (eg the remote file has been fully downloaded over the network) so that passthrough can be switched on even with the application having open fds.

Indeed, being able to switch to passthrough for new fds is a big win, which is why I also changed to this method in the current implementation.

Quoted 27 lines
Quoted 9 lines

Adding inode passthrough ops does not play well with this mode this is why it requires the lookup-to-forget lifecycle, but I see no reason kill the open-to-close lifecycle as it seems useful for servers that mostly care about I/O passthrough.

w.r.t expanded struct fuse_backing_map, at the moment it does not sound right to me to map a backing id X with more than a single backing file or device.

My thinking with the struct fuse_backing_map expansion was that future ioctls for managing backing files (eg attaching a backing file to a live inode or swapping one backing file for another) might benefit from a shared struct instead of needing to add new structs for each type.

Quoted 4 lines

I think that the maps of fuse_file to multiple fuse_backing is better handled by another layer of mapping (sort of a backing_mapper) which is what famfs and iomap need, but it is just my initial thoughts.

I like this idea a lot, it makes sense to me.

Quoted 3 lines

Joanne, if you are a concrete use case for that, let's try to see
how a design to this use case may look like.

Quoted 9 lines

The internal team I'm working with is planning to start working on their server in H2. They will run some benchmarks to see if current passthrough and/or io-uring+zero-copy might get them the performance gains they need, or whether they will still need the set of advanced passthrough features they requested. I'll keep you posted once there's performance data and if there is still a need for the more advanced passthrough features (file mapping to multiple backing files, backing files dynamically changing, enabling passthrough on a live inode), I'll draft up a design doc to get your thoughts on.

Quoted 7 lines

Sounds good.
FYI, some of those issues we try to solve at application layer. Our HSM application always tries to keep the inode itself across state changes.
For example, by using clone_file_range() to fill in content. With xfs, we could also try to use atomic swap extents, if file content can change, instead of changing inode, but we did not try this yet.

Thanks for the info. This is useful, I'll look into seeing if some of these requests can be solved at the application layer.

Quoted 8 lines
Quoted 5 lines

In the meantime,
I'm planning to send out v2 of the inode metadata passthrough patchset sometime this week or next with your feedback from v1.

Cool. Let's get this rolling.
I need to get my test setup back in shape.

What is the test setup you are using? fstests with passthrough_hp or are there other tests in addition to that? If there's other tests you'd like me to run besides fstests, please let me know and I'd be happy to do that.

When I ran fstests yesterday with passthrough_hp, I saw generic 683, 684, and 685 (dropping suid/sgid on fallocate, fpunch, and fzero) start passing, but beyond that I didn't see any improvements (or regressions).

Thanks,
Joanne

Quoted 8 lines

There were a bunch of fstest failing last time I tested passthrough, mostly related to metadata cache coherency.
I was hoping that GETATTR passthrough could solve some of them. If you have seen any improvements (or regressions) in fstests please share this info with the patch set.

Thanks,
Amir.

#20 lore raw
Quoted 1 line · Joanne Koong

...

Quoted 4 lines
Quoted 2 lines

Cool. Let's get this rolling.
I need to get my test setup back in shape.

What is the test setup you are using? fstests with passthrough_hp or

Yes. *was* using while I developed the passthrough patches. Didn't run it for a while.

Quoted 3 lines

are there other tests in addition to that? If there's other tests you'd like me to run besides fstests, please let me know and I'd be happy to do that.

Nothing beyond the libfuse test_passthrough_hp sanity test and fstests with passthough_hp.

Quoted 5 lines

When I ran fstests yesterday with passthrough_hp, I saw generic 683, 684, and 685 (dropping suid/sgid on fallocate, fpunch, and fzero) start passing, but beyond that I didn't see any improvements (or regressions).

That's the last report I have on record:
https://lore.kernel.org/linux-fsdevel/CAOQ4uxh=cLySge6htd+DnJrqAKF=C_oJYfVrbpvQGik0wR-+iw@mail.gmail.com/

It speaks of some tests that are
"failing because of missing mtime/atime/ctime updates" and of "failure to strip suid/sgid bits in some cases"

I had imagined that the passthrough of GETATTR would fix some of those failing tests and to some extent, this is in line with your report.

Attribute cache and passthrough is an unfinished business. If we could force passthrough of GETATTR when inode is in passthrough iomode, I wouldn't mind.

Thanks,
Amir.

#21 lore raw
Quoted 20 lines · Amir Goldstein
Quoted 1 line · Joanne Koong

...

Quoted 4 lines
Quoted 2 lines

Cool. Let's get this rolling.
I need to get my test setup back in shape.

What is the test setup you are using? fstests with passthrough_hp or

Yes. *was* using while I developed the passthrough patches. Didn't run it for a while.

Quoted 3 lines

are there other tests in addition to that? If there's other tests you'd like me to run besides fstests, please let me know and I'd be happy to do that.

Nothing beyond the libfuse test_passthrough_hp sanity test and fstests with passthough_hp.

Quoted 1 line
Quoted 7 lines
Quoted 4 lines

When I ran fstests yesterday with passthrough_hp, I saw generic 683, 684, and 685 (dropping suid/sgid on fallocate, fpunch, and fzero) start passing, but beyond that I didn't see any improvements (or regressions).

That's the last report I have on record:
https://lore.kernel.org/linux-fsdevel/CAOQ4uxh=cLySge6htd+DnJrqAKF=C_oJYfVrbpvQGik0wR-+iw@mail.gmail.com/

Thanks for the link. I had compared my test results against runs on passthrough_hp --nopassthrough.

Comparing them now to baseline passthrough_hp (no inode passthrough commits) on the tests you listed:
- on baseline passthrough_hp, I'm seeing generic/193 and
generic/355 now pass compared to when you ran them in 2023. generic/080, generic/120, and generic/215 still fail
- with the inode passthrough commits, I see generic/080 and
generic/215 now pass. generic/120 is still failing, which I think is because the backing mount doesn't have noatime but the fuse mount does, so getattr passthrough to the backing file will fetch the updated access time.

Thanks,
Joanne

Quoted 14 lines

It speaks of some tests that are
"failing because of missing mtime/atime/ctime updates" and of "failure to strip suid/sgid bits in some cases"

I had imagined that the passthrough of GETATTR would fix some of those failing tests and to some extent, this is in line with your report.

Attribute cache and passthrough is an unfinished business. If we could force passthrough of GETATTR when inode is in passthrough iomode, I wouldn't mind.

Thanks,
Amir.

#22 lore raw
Quoted 33 lines · Joanne Koong
Quoted 20 lines · Amir Goldstein
Quoted 1 line · Joanne Koong

...

Quoted 4 lines
Quoted 2 lines

Cool. Let's get this rolling.
I need to get my test setup back in shape.

What is the test setup you are using? fstests with passthrough_hp or

Yes. *was* using while I developed the passthrough patches. Didn't run it for a while.

Quoted 3 lines

are there other tests in addition to that? If there's other tests you'd like me to run besides fstests, please let me know and I'd be happy to do that.

Nothing beyond the libfuse test_passthrough_hp sanity test and fstests with passthough_hp.

Quoted 1 line
Quoted 7 lines
Quoted 4 lines

When I ran fstests yesterday with passthrough_hp, I saw generic 683, 684, and 685 (dropping suid/sgid on fallocate, fpunch, and fzero) start passing, but beyond that I didn't see any improvements (or regressions).

That's the last report I have on record:
https://lore.kernel.org/linux-fsdevel/CAOQ4uxh=cLySge6htd+DnJrqAKF=C_oJYfVrbpvQGik0wR-+iw@mail.gmail.com/

Thanks for the link. I had compared my test results against runs on passthrough_hp --nopassthrough.

Quoted 11 lines

Comparing them now to baseline passthrough_hp (no inode passthrough commits) on the tests you listed:
- on baseline passthrough_hp, I'm seeing generic/193 and
generic/355 now pass compared to when you ran them in 2023. generic/080, generic/120, and generic/215 still fail
- with the inode passthrough commits, I see generic/080 and
generic/215 now pass. generic/120 is still failing, which I think is because the backing mount doesn't have noatime but the fuse mount does, so getattr passthrough to the backing file will fetch the updated access time.

Nice! This is what I would have expected.

Thanks,
Amir.

#23 lore raw
In reply to Miklos Szeredi
Quoted 14 lines · Miklos Szeredi
Quoted 10 lines · Joanne Koong

If it's mostly to clean up interfaces that are now clunky/stale, I wonder if now is the best time given that, as Amir phrased it in [1], "FUSE is experiencing a renaissance of new features and protocol enhancements", and I'm not sure if we know yet if these new interfaces will also have some things we'll have wished in hindsight we had done differently, especially since some of these new features are still actively evolving and gaining new capabilities (including io-uring and passthrough). Would it make sense to let these new features and protocol enhancements evolve and stabilize first before baking them into fusex?

Quoted 4 lines

Excellent point.

Currently fusex concerns only the filesystem layer, and is agnostic to the transport layer (except maybe the SYNC_INIT thing). That doesn't

As soon as fusex lands, doesn't this mean with the linux backwards compatibility policy that any user-facing transport layer decisions are also locked in? Or is the uapi for it still able to change later because fusex is marked as "experimental"?

Quoted 53 lines

mean that we shouldn't have the same API cleanup in the transport layer as well. For example, I'd love to get rid of splice on /dev/fuse, which I feel is more burden than blessing. And I guess fuse-uring could also do with an API cleanup.

As to when and how this should be done? I guess that's up to you and Bernd to decide.

But I feel that we do need to have more high level discussion of APIs, if we want to avoid repeating past mistakes.

Quoted 6 lines
Quoted 5 lines

- only no-open mode
- new requests:
+ FUSE_LOOKUP_ROOT - return nodeid of root
+ FUSE_LOOKUPX - FUSE_LOOKUP without the getattr

Quoted 9 lines

Could you explain why it's preferred to have lookup disentangled from attributes, eg why compounding lookup + attribute fetching is preferred over embedding the statx directly inside the lookup request without compounding? it was brought up in Luis's lookup handles patchset [2] but it's still a bit unclear to me why compounding the two is better. I think every operation/request that returns a struct fuse_entry_out uses the attributes immediately after through fuse_iget(..., &outarg->attr, ...) or fuse_change_attributes(..., &outarg.attr, ...),

That's one argument for decoupling: why include the same fields (attributes, validity) in several ops (LOOKUP_ROOT, LOOKUPX, MKOBJX) if we already have a separate one to do that? It simplifies the protocol. If we started with decoupled GETATTR, then we wouldn't be taking about adding these X versions, because it would be a simple GETATTR -> STATX conversion.

There's also an interesting side effect of coupling STATX with LOOKUP, which is inherently racy:

Thread A:
send LOOKUP + STATX for /fuse/dir1/filea
[server performs lookup and statx, sends reply]

Thread B:
send SETATTR for /fuse/dir2/fileb, which is hard linked to /fuse/dir1/filea evict inode for /fuse/dir2/fileb

Thread A:
looks up inode for /fuse/dir1/filea in cache, doesn't find any creates inode, fills stale attributes

This issue is currently solved with fc->evict_ctr in regular fuse, but works fine without any additional checks in fusex because of the decoupled ops.

Quoted 8 lines

Of course, it is probably desirable to compound LOOKUP with STATX despite this, but it gives the implementation more leeway (e.g. on a different OS).

To conclude: separating out STATX from LOOKUPX, etc. makes the protocol:

a) simpler
b) more flexible

This makes sense to me, thanks for the explanation.

Thanks,
Joanne

#24 lore raw
Quoted 17 lines · Joanne Koong
Quoted 16 lines · Miklos Szeredi
Quoted 10 lines · Joanne Koong

If it's mostly to clean up interfaces that are now clunky/stale, I wonder if now is the best time given that, as Amir phrased it in [1], "FUSE is experiencing a renaissance of new features and protocol enhancements", and I'm not sure if we know yet if these new interfaces will also have some things we'll have wished in hindsight we had done differently, especially since some of these new features are still actively evolving and gaining new capabilities (including io-uring and passthrough). Would it make sense to let these new features and protocol enhancements evolve and stabilize first before baking them into fusex?

Excellent point.

Quoted 7 lines
Quoted 2 lines

Currently fusex concerns only the filesystem layer, and is agnostic to the transport layer (except maybe the SYNC_INIT thing). That doesn't

As soon as fusex lands, doesn't this mean with the linux backwards compatibility policy that any user-facing transport layer decisions are also locked in? Or is the uapi for it still able to change later because fusex is marked as "experimental"?

Create a permissions checking predicate, ala:

bool fuse_iomap_enabled(void)

{
	/* Don't let anyone touch iomap until the end of the patchset. */
	return false;

	/*
	 * There are fears that a fuse+iomap server could somehow DoS the
	 * system by doing things like going out to lunch during a writeback
	 * related iomap request.  Only allow iomap access if the fuse server
	 * has rawio capabilities since those processes can mess things up
	 * quite well even without our help.
	 */
	return enable_iomap && has_capability_noaudit(current, CAP_SYS_RAWIO);
}

and a default N Kconfig option, and then the only people who can actually use the uabi are fuse developers who are willing to patch and run their own kernels.

--D

Quoted 67 lines
Quoted 53 lines

mean that we shouldn't have the same API cleanup in the transport layer as well. For example, I'd love to get rid of splice on /dev/fuse, which I feel is more burden than blessing. And I guess fuse-uring could also do with an API cleanup.

As to when and how this should be done? I guess that's up to you and Bernd to decide.

But I feel that we do need to have more high level discussion of APIs, if we want to avoid repeating past mistakes.

Quoted 6 lines
Quoted 5 lines

- only no-open mode
- new requests:
+ FUSE_LOOKUP_ROOT - return nodeid of root
+ FUSE_LOOKUPX - FUSE_LOOKUP without the getattr

Quoted 9 lines

Could you explain why it's preferred to have lookup disentangled from attributes, eg why compounding lookup + attribute fetching is preferred over embedding the statx directly inside the lookup request without compounding? it was brought up in Luis's lookup handles patchset [2] but it's still a bit unclear to me why compounding the two is better. I think every operation/request that returns a struct fuse_entry_out uses the attributes immediately after through fuse_iget(..., &outarg->attr, ...) or fuse_change_attributes(..., &outarg.attr, ...),

That's one argument for decoupling: why include the same fields (attributes, validity) in several ops (LOOKUP_ROOT, LOOKUPX, MKOBJX) if we already have a separate one to do that? It simplifies the protocol. If we started with decoupled GETATTR, then we wouldn't be taking about adding these X versions, because it would be a simple GETATTR -> STATX conversion.

There's also an interesting side effect of coupling STATX with LOOKUP, which is inherently racy:

Thread A:
send LOOKUP + STATX for /fuse/dir1/filea
[server performs lookup and statx, sends reply]

Thread B:
send SETATTR for /fuse/dir2/fileb, which is hard linked to /fuse/dir1/filea evict inode for /fuse/dir2/fileb

Thread A:
looks up inode for /fuse/dir1/filea in cache, doesn't find any creates inode, fills stale attributes

This issue is currently solved with fc->evict_ctr in regular fuse, but works fine without any additional checks in fusex because of the decoupled ops.

Quoted 8 lines

Of course, it is probably desirable to compound LOOKUP with STATX despite this, but it gives the implementation more leeway (e.g. on a different OS).

To conclude: separating out STATX from LOOKUPX, etc. makes the protocol:

a) simpler
b) more flexible

This makes sense to me, thanks for the explanation.

Thanks,
Joanne

#25 lore raw
In reply to Miklos Szeredi
Quoted 13 lines · Miklos Szeredi

This stands for "fuse extended/experimental".

The purpose is to provide a clean base for big features like the FUSE_IOMAP api.

It's also a good way to try new stuff like file handles and compound requests without the risk of breaking something in the large and complex fuse codebase.

Whether these features will be migrated back into the main fuse codebase, or fusex is going to end up as a major version update is still up in the air.

[...]

fs/fuse/fusex.c line 27

Quoted 2 lines
fs/fuse/fusex.c new file +2 -0
+#define ADD_IN_ARG_ZERO(args) \
+	ADD_IN_ARG(args, 0, NULL)

Hi Miklos,

could we avoid that? If we don't want to change existing op codes, could we avoid it at least for any new op code?

DDN currently has a problem with

LOOKUP
STATX

and fuse_permission() calling into getattr.

Issues come from DLM and distributed metadata operations. It is relatively simple to handle a total dentry/inode invalidation when a file gets removed or moved to another dir. The really hard part are time stamps and file size. I.e. most of the time fuse-client attributes are just in state invalid. But then for single client access it also makes sense to have attributes cached for ininity. So DLM invalidates attributes if things change on a remote system.

Issue are now file creations of multiple clients in the same dir - the parent does not have updated file attributes and actually does not need to have them either for plain revalidate or permission attribute fetch.

For perm getattr it is relatively straight forward to use statx and add in sx_flags that the server has to fetch. It can respond with more and then cache time stamp gets updated. If some attr missing, time stamp is not changed (detable it should be set to invalid)

Revalidate is harder, in the range to impossible with the curent FUSE_LOOKUP call. There is no header, where a flag could get passed in that it is a revalidate and not all attribtes are needed.

From my point of view, every single op code should have their own u32 feature flags field, so that an op code is easy to extend and without the need to update the global flags.

[...]

fs/fuse/fusex.c line 659

Quoted 10 lines
fs/fuse/fusex.c new file +10 -0
+
+static struct inode *fusex_do_lookup(struct inode *base, const struct qstr *name)
+{
+	FUSE_ARGS(args);
+	struct fuse_entryx_out outarg;
+	struct fusex_id id;
+	int err;
+
+	args.opcode = FUSE_LOOKUPX;
+	ADD_IN_ARG_ZERO(args);

Here is would be good to have something like

#define FUSE_LOOKUPX_IN_IS_REVALIDATE (1u << )

struct fuse_lookupx_in {
int32_t op_flags; /* op code feature flags */
int32_t lookup_flags; /* so far FUSE_LOOKUPX_IN_IS_REVALIDATE */
};

Thanks,
Bernd

maintainer
#26 lore raw
Quoted 5 lines · Bernd Schubert

fs/fuse/fusex.c line 27

Quoted 2 lines
fs/fuse/fusex.c new file +2 -0
+#define ADD_IN_ARG_ZERO(args) \
+     ADD_IN_ARG(args, 0, NULL)

could we avoid that? If we don't want to change existing op codes, could we avoid it at least for any new op code?

Good point.

What we really want is fixed size headers, and what better time to do that with fusex.

Do you see any issue with just taking the 128 + 128 bytes that fuse-uring is using and padding the headers with zeroes?

fuse_setstatx_in goes over the limit, but we can just move the statx part to the payload.

Thanks,
Miklos

#27 lore raw
In reply to Miklos Szeredi

Hi Miklos!

I'm not sure if you have a more up-to-date version of your patch (or patchset, as you mentioned you'd be splitting it). But here's a small issue I have fixed locally a while back, when I started playing with fusex:

On Wed, Apr 29 2026, Miklos Szeredi wrote:

<snip>

fs/fuse/fusex.c line 308

Quoted 11 lines
fs/fuse/fusex.c new file +11 -0
+static struct inode *fusex_get_inode(struct super_block *sb, struct fusex_id *id)
+{
+	struct inode *inode = fusex_iget(sb, id);
+	struct fuse_statx_out statx;
+	int err;
+
+	if (!inode)
+		return ERR_PTR(-ENOMEM);
+
+	if (inode_state_read_once(inode) & I_NEW) {
+		inode_state_set(inode, I_DONTCACHE);

inode_state_set() assumes the caller holds inode->i_lock, so we will see a kernel splat from a lockdep assertion.

Cheers,

Signature · 1 line
Luís
Quoted 11 lines
fs/fuse/fusex.c new file +11 -0
+		err = fusex_send_statx(inode, &statx);
+		if (err) {
+			discard_new_inode(inode);
+			return ERR_PTR(err);
+		}
+		fusex_set_attr(inode, &statx.stat);
+		fusex_init_inode(inode);
+		unlock_new_inode(inode);
+	}
+	return inode;
+}