fuse: add fusex filesystem
Subsystems (2)
PATCH 1/1 fuse: add fusex filesystem
This stands for "fuse extended/experimental".
The purpose is to provide a clean base for big features like the FUSE_IOMAP api.
It's also a good way to try new stuff like file handles and compound requests without the risk of breaking something in the large and complex fuse codebase.
Whether these features will be migrated back into the main fuse codebase, or fusex is going to end up as a major version update is still up in the air.
Major differences from regular fuse:
- local filesystem mode only
- only synchronous FUSE_INIT is supported
- only no-open mode
- new requests:
+ FUSE_LOOKUP_ROOT - return nodeid of root
+ FUSE_LOOKUPX - FUSE_LOOKUP without the getattr
+ FUSE_MKOBJX - merged FUSE_MKNOD, MKDIR, SYMLINK and TMPFILE
+ FUSE_SETSTATX - extended version of FUSE_SETATTR
Missing features:
- file handles / export ops
- compound requests
- xattr caching
- fileattr
- fiemap
- ioctl
- copy_file_range
- lazy dir open
Test server can be found at:
https://github.com/szmi/fuse-utils
---
Patch is against
git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git#for-next
fs/fuse/Makefile | 2 +- fs/fuse/args.h | 1 + fs/fuse/dev.c | 13 +- fs/fuse/dir.c | 4 +- fs/fuse/fuse_i.h | 8 + fs/fuse/fusex.c | 1751 +++++++++++++++++++++++++++++++++++++ fs/fuse/fusex.h | 4 + fs/fuse/inode.c | 16 +- include/uapi/linux/fuse.h | 36 + 9 files changed, 1826 insertions(+), 9 deletions(-) create mode 100644 fs/fuse/fusex.c create mode 100644 fs/fuse/fusex.h
fs/fuse/Makefile +1 -1
@@ -12,7 +12,7 @@ obj-$(CONFIG_VIRTIO_FS) += virtiofs.o fuse-y := trace.o # put trace.o first so we see ftrace errors sooner fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o req_timeout.o req.o -fuse-y += poll.o notify.o +fuse-y += poll.o notify.o fusex.o fuse-y += iomode.o fuse-$(CONFIG_FUSE_DAX) += dax.o fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o backing.o
fs/fuse/args.h +1 -0
@@ -35,6 +35,7 @@ struct fuse_args { bool out_pages:1; bool user_pages:1; bool out_argvar:1; + bool out_var_alloc:1; bool page_zeroing:1; bool page_replace:1; bool may_block:1;
fs/fuse/dev.c +11 -2
@@ -1847,15 +1847,24 @@ int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args, reqsize +=fuse_len_args (args->out_numargs, args->out_args); -if (reqsize < nbytes || (reqsize > nbytes && !args->out_argvar)) +if (reqsize < nbytes)return -EINVAL ; -else if (reqsize > nbytes) { + +if (args->out_argvar) {struct fuse_arg *lastarg = &args->out_args[args->out_numargs-1 ];unsigned diffsize = reqsize - nbytes;if (diffsize > lastarg->size)return -EINVAL ; lastarg->size -= diffsize; + +if (args->out_var_alloc) { + lastarg->value =kvmalloc (lastarg->size,GFP_KERNEL ); +if (!lastarg->value) +return -ENOMEM ; + } + }else if (reqsize > nbytes) { +return -EINVAL ; }return fuse_copy_args (cs, args->out_numargs, args->out_pages, args->out_args, args->page_zeroing);
fs/fuse/dir.c +2 -2
@@ -542,7 +542,7 @@ int fuse_valid_type(int m)S_ISBLK (m) ||S_ISFIFO (m) ||S_ISSOCK (m); } -static bool fuse_valid_size (u64 size) +bool fuse_valid_size (u64 size) {return size <=LLONG_MAX ; }
@@ -2485,7 +2485,7 @@ static int fuse_symlink_read_folio(struct file *null, struct folio *folio)return err; } -static const struct address_space_operations fuse_symlink_aops = { +const struct address_space_operations fuse_symlink_aops = { .read_folio = fuse_symlink_read_folio, };
fs/fuse/fuse_i.h +8 -0
@@ -69,6 +69,9 @@ extern struct mutex fuse_mutex;extern unsigned int max_user_bgreq;extern unsigned int max_user_congthresh; +extern struct kmem_cache *fuse_inode_cachep; +extern const struct address_space_operations fuse_symlink_aops; +struct fuse_forget_link ;/**
@@ -911,6 +914,8 @@ struct inode *fuse_iget(struct super_block *sb, u64 nodeid,int fuse_lookup_name (struct super_block *sb,u64 nodeid,const struct qstr *name,struct fuse_entry_out *outarg,struct inode **inode); +void fuse_umount_begin (struct super_block *sb); +/* * Initialize READ or READDIR request */
@@ -1102,6 +1107,7 @@ void fuse_ctl_remove_conn(struct fuse_conn *fc);* Is file type valid? */ int fuse_valid_type (int m); +bool fuse_valid_size (u64 size);bool fuse_invalid_attr (struct fuse_attr *attr);
@@ -1204,6 +1210,8 @@ struct posix_acl *fuse_get_acl(struct mnt_idmap *idmap,int fuse_set_acl (struct mnt_idmap *,struct dentry *dentry,struct posix_acl *acl,int type); +void fuse_convert_statfs (struct kstatfs *stbuf,struct fuse_kstatfs *attr); +/* readdir.c */ int fuse_readdir (struct file *file,struct dir_context *ctx);
fs/fuse/fusex.c new file +1751 -0
1,751 lines not on this page · raw message
fs/fuse/fusex.h new file +4 -0
@@ -0,0 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +int fusex_init (void ); +void fusex_cleanup (void );
fs/fuse/inode.c +12 -4
@@ -8,6 +8,7 @@#include "dev.h" #include "fuse_i.h" +#include "fusex.h" #include <linux/dax.h> #include <linux/pagemap.h>
@@ -31,7 +32,7 @@ MODULE_AUTHOR("Miklos Szeredi <[email protected]>");MODULE_DESCRIPTION ("Filesystem in Userspace" );MODULE_LICENSE ("GPL" ); -static struct kmem_cache *fuse_inode_cachep; +struct kmem_cache *fuse_inode_cachep;struct list_head fuse_conn_list;DEFINE_MUTEX (fuse_mutex);
@@ -605,7 +606,7 @@ void fuse_unlock_inode(struct inode *inode, bool locked)mutex_unlock (&get_fuse_inode (inode)->mutex); } -static void fuse_umount_begin (struct super_block *sb) +void fuse_umount_begin (struct super_block *sb) {struct fuse_conn *fc =get_fuse_conn_super (sb);
@@ -631,7 +632,7 @@ static void fuse_send_destroy(struct fuse_mount *fm) } } -static void convert_fuse_statfs (struct kstatfs *stbuf,struct fuse_kstatfs *attr) +void fuse_convert_statfs (struct kstatfs *stbuf,struct fuse_kstatfs *attr) { stbuf->f_type =FUSE_SUPER_MAGIC ; stbuf->f_bsize = attr->bsize;
@@ -667,7 +668,7 @@ static int fuse_statfs(struct dentry *dentry, struct kstatfs *buf) args.out_args[0 ].value = &outarg; err =fuse_simple_request (fm, &args);if (!err) -convert_fuse_statfs (buf, &outarg.st); +fuse_convert_statfs (buf, &outarg.st);return err; }
@@ -2160,6 +2161,10 @@ static int __init fuse_init(void)if (res) goto err_sysfs_cleanup; + res =fusex_init (); +if (res) + goto err_ctl_cleanup; +fuse_dentry_tree_init ();sanitize_global_limit (&max_user_bgreq);
@@ -2167,6 +2172,8 @@ static int __init fuse_init(void)return 0 ; +err_ctl_cleanup: +fuse_ctl_cleanup (); err_sysfs_cleanup:fuse_sysfs_cleanup (); err_dev_cleanup:
@@ -2182,6 +2189,7 @@ static void __exit fuse_exit(void)pr_debug ("exit\n" );fuse_dentry_tree_cleanup (); +fusex_cleanup ();fuse_ctl_cleanup ();fuse_sysfs_cleanup ();fuse_fs_cleanup ();
include/uapi/linux/fuse.h +36 -0
@@ -663,6 +663,10 @@ enum fuse_opcode {FUSE_TMPFILE =51 ,FUSE_STATX =52 ,FUSE_COPY_FILE_RANGE_64 =53 , +FUSE_LOOKUP_ROOT =54 , +FUSE_LOOKUPX =55 , +FUSE_MKOBJX =56 , +FUSE_SETSTATX =58 ,/* CUSE specific operations */ CUSE_INIT = 4096,
@@ -700,6 +704,20 @@ struct fuse_entry_out {struct fuse_attr attr; }; +/* +* entryx flags +* FUSE_ENTRYX_NEGATIVE: file does not exist, can cache this result +*/ +#define FUSE_ENTRYX_NEGATIVE (1 << 0) + +struct fuse_entryx_out { +uint64_t nodeid; +uint64_t entry_valid; +uint32_t entry_valid_nsec; +uint32_t flags; +uint64_t spare; +}; +struct fuse_forget_in {uint64_t nlookup; };
@@ -754,6 +772,17 @@ struct fuse_mknod_in {uint32_t padding; }; +enum fuse_mkobjx_flags { +FUSE_MKOBJX_TMPFILE =1 <<0 , +}; + +struct fuse_mkobjx_in { +struct fuse_statx stat; +uint32_t namesize; +uint32_t flags; +uint64_t spare[7 ]; +}; +struct fuse_mkdir_in {uint32_t mode;uint32_t umask;
@@ -792,6 +821,13 @@ struct fuse_setattr_in {uint32_t unused5; }; +struct fuse_setstatx_in { +uint64_t fh; +uint32_t flags; +uint32_t reserved; +struct fuse_statx stat; +}; +struct fuse_open_in {uint32_t flags;uint32_t open_flags;/* FUSE_OPEN_... */
Signature · 1 line
2.53.0
Quoted 35 lines · Miklos Szeredi
This stands for "fuse extended/experimental".
The purpose is to provide a clean base for big features like the
FUSE_IOMAPapi.It's also a good way to try new stuff like file handles and compound requests without the risk of breaking something in the large and complex fuse codebase.
Whether these features will be migrated back into the main fuse codebase, or fusex is going to end up as a major version update is still up in the air.
Major differences from regular fuse:
- local filesystem mode only
- only synchronousFUSE_INITis supported
- only no-open mode
- new requests:
+FUSE_LOOKUP_ROOT- return nodeid of root
+FUSE_LOOKUPX-FUSE_LOOKUPwithout the getattr
+FUSE_MKOBJX- mergedFUSE_MKNOD, MKDIR, SYMLINK and TMPFILE
+FUSE_SETSTATX- extended version ofFUSE_SETATTRMissing features:
- file handles / export ops
- compound requests
- xattr caching
- fileattr
- fiemap
- ioctl
-copy_file_range
- lazy dir open
Quoted 7 lines
Test server can be found at:
https://github.com/szmi/fuse-utils
Signed-off-by Miklos Szeredi <[email protected]>---
...
Hi Miklos,
I really like the new shiny fusex. It just has one heavy drawback for me. It is not set to support remote file systems.
I have a coupe of questions that I would like to ask here on the list.
What was the reasoning behind this? Complexity, lack of interest from the community/users of fuse?
Where do you see the biggest challenge for this. To me, it doesn't look impossible to add that, but I'm sure I'm missing a lot in this context.
Thanks,
Horst
Quoted 11 lines · Horst Birthelmer
Quoted 9 lines · Miklos Szeredi
This stands for "fuse extended/experimental".
The purpose is to provide a clean base for big features like the
FUSE_IOMAPapi.It's also a good way to try new stuff like file handles and compound requests without the risk of breaking something in the large and complex fuse codebase.
Quoted 7 lines
Quoted 7 lines
Whether these features will be migrated back into the main fuse codebase, or fusex is going to end up as a major version update is still up in the air.
Major differences from regular fuse:
- local filesystem mode only
What about things like?FUSE_HANDLE_KILLPRIV | FUSE_HANDLE_KILLPRIV_V2FUSE_ATOMIC_O_TRUNC
Does local-only mean that all those (and default_permissions) are handled by the kernel?
Quoted 11 lines
Quoted 11 lines
- only synchronous
FUSE_INITis supported
- only no-open mode
- new requests:
+FUSE_LOOKUP_ROOT- return nodeid of root
+FUSE_LOOKUPX-FUSE_LOOKUPwithout the getattr
+FUSE_MKOBJX- mergedFUSE_MKNOD, MKDIR, SYMLINK and TMPFILE
+FUSE_SETSTATX- extended version ofFUSE_SETATTRMissing features:
- file handles / export ops
Need to sort out this API that grew odd organically over the years:
FUSE_EXPORT_SUPPORT | FUSE_NO_EXPORT_SUPPORT
The problem is that libfuse declares FUSE_EXPORT_SUPPORT on behalf of high level filesystems which do not really provide thse persistent NFS file handle guarantees.
I see no reason for fusex to not support lookup by ".", which is anyway needed for "reconnect", so a fresh FUSEX_NFS_EXPORT opt-in for "re-export to NFS" may make more sense.
Thanks,
Amir.
Miklos Szeredi
maintainerQuoted 7 lines · Amir Goldstein
Quoted 5 lines · Horst Birthelmer
Quoted 2 lines · Miklos Szeredi
Major differences from regular fuse:
Quoted 8 lines
Quoted 1 line
Quoted 1 line
- local filesystem mode only
What about things like?
FUSE_HANDLE_KILLPRIV|FUSE_HANDLE_KILLPRIV_V2FUSE_ATOMIC_O_TRUNCDoes local-only mean that all those (and
default_permissions) are handled by the kernel?
Yes.
Quoted 3 lines
I see no reason for fusex to not support lookup by ".", which is anyway needed for "reconnect", so a fresh
FUSEX_NFS_EXPORTopt-in for "re-export to NFS" may make more sense.
Why add reconnect? If server supports persistent file handles, and all req's get the file handle, then no state (nodeid) needs to be stored by the server.
Thanks,
Miklos
Quoted 9 lines · Miklos Szeredi
Quoted 4 lines
I see no reason for fusex to not support lookup by ".", which is anyway needed for "reconnect", so a fresh
FUSEX_NFS_EXPORTopt-in for "re-export to NFS" may make more sense.Why add reconnect? If server supports persistent file handles, and all req's get the file handle, then no state (nodeid) needs to be stored by the server.
Yes, we are saying the same thing.
What I am saying is that legacy fuse has FUSE_EXPORT_SUPPORT which means server supports lookup of "." which libfuse always enabled and then we added FUSE_NO_EXPORT_SUPPORT to opt-out of nfs export.
I am saying that in FUSEX, lookup "." support is implied. And that nfs export should always be opt-in (FUSEX_NFS_EXPORT).
Thanks,
Amir.
Quoted 11 lines · Amir Goldstein
Quoted 9 lines · Miklos Szeredi
Quoted 4 lines
I see no reason for fusex to not support lookup by ".", which is anyway needed for "reconnect", so a fresh
FUSEX_NFS_EXPORTopt-in for "re-export to NFS" may make more sense.Why add reconnect? If server supports persistent file handles, and all req's get the file handle, then no state (nodeid) needs to be stored by the server.
Quoted 7 lines
Yes, we are saying the same thing.
What I am saying is that legacy fuse hasFUSE_EXPORT_SUPPORTwhich means server supports lookup of "." which libfuse always enabled and then we addedFUSE_NO_EXPORT_SUPPORTto opt-out of nfs export.I am saying that in FUSEX, lookup "." support is implied. And that nfs export should always be opt-in (
FUSEX_NFS_EXPORT).
I'm confused here, why do we need lookup of "." without NFS export? And which file system actually supports lookup of "."?
Shouldn't all that be scratched in favor of LOOKUP_HANDLE?
Thanks,
Bernd
Quoted 20 lines · Bernd Schubert
Quoted 16 lines · Amir Goldstein
Quoted 9 lines · Miklos Szeredi
Quoted 4 lines
I see no reason for fusex to not support lookup by ".", which is anyway needed for "reconnect", so a fresh
FUSEX_NFS_EXPORTopt-in for "re-export to NFS" may make more sense.Why add reconnect? If server supports persistent file handles, and all req's get the file handle, then no state (nodeid) needs to be stored by the server.
Yes, we are saying the same thing.
What I am saying is that legacy fuse hasFUSE_EXPORT_SUPPORTwhich means server supports lookup of "." which libfuse always enabled and then we addedFUSE_NO_EXPORT_SUPPORTto opt-out of nfs export.
Quoted 6 lines
Quoted 2 lines
I am saying that in FUSEX, lookup "." support is implied. And that nfs export should always be opt-in (
FUSEX_NFS_EXPORT).I'm confused here, why do we need lookup of "." without NFS export? And which file system actually supports lookup of "."?
Shouldn't all that be scratched in favor ofLOOKUP_HANDLE?
Yes. This is exactly what I meant by lookup "." support is implied.
AFAICT, libfuse sets CAP_EXPORT_SUPPORT unconditionally:
fuse_set_feature_flag(conn, FUSE_CAP_EXPORT_SUPPORT);
ALL of the high level fuse fs are allowed to be exported by NFS, which is often a very bad idea.
Thanks,
Amir.
Quoted 28 lines · Amir Goldstein
Quoted 20 lines · Bernd Schubert
Quoted 16 lines · Amir Goldstein
Quoted 9 lines · Miklos Szeredi
Quoted 4 lines
I see no reason for fusex to not support lookup by ".", which is anyway needed for "reconnect", so a fresh
FUSEX_NFS_EXPORTopt-in for "re-export to NFS" may make more sense.Why add reconnect? If server supports persistent file handles, and all req's get the file handle, then no state (nodeid) needs to be stored by the server.
Yes, we are saying the same thing.
What I am saying is that legacy fuse hasFUSE_EXPORT_SUPPORTwhich means server supports lookup of "." which libfuse always enabled and then we addedFUSE_NO_EXPORT_SUPPORTto opt-out of nfs export.Quoted 6 lines
Quoted 2 lines
I am saying that in FUSEX, lookup "." support is implied. And that nfs export should always be opt-in (
FUSEX_NFS_EXPORT).I'm confused here, why do we need lookup of "." without NFS export? And which file system actually supports lookup of "."?
Shouldn't all that be scratched in favor ofLOOKUP_HANDLE?
Quoted 8 lines
Yes. This is exactly what I meant by lookup "." support is implied.
AFAICT, libfuse sets
CAP_EXPORT_SUPPORTunconditionally:fuse_set_feature_flag(conn, FUSE_CAP_EXPORT_SUPPORT);ALL of the high level fuse fs are allowed to be exported by NFS, which is often a very bad idea.
I think that predates me and is in the came category as FUSE_CAP_AUTO_INVAL_DATA. Initially I thought it gets really complicated and I need to add another ABI/API compat layer to encode into struct fuse_session which FUSE_USE_VERSION the application uses, but then in the discussion with Darrick we got the idea to put it into the existing padding
fuse_session_new_fn(struct fuse_args *args, const struct fuse_lowlevel_ops *op,
size_t op_size, void *userdata)
{
struct libfuse_version version = {
.major = FUSE_MAJOR_VERSION, .minor = FUSE_MINOR_VERSION, .hotfix = FUSE_HOTFIX_VERSION, .padding = 0
};
return fuse_session_new_versioned(args, op, op_size, &version,
userdata);
}
So idea and suggestion Miklos for FUSE_CAP_AUTO_INVAL_DATA was to chage the default based on FUSE_USE_VERSION. We can do the same for FUSE_CAP_EXPORT_SUPPORT.
The initial patch to add in that version into libfuse as another function argument got really complex in the high level layer and then QA team at DDN figure out that disabling FUSE_CAP_AUTO_INVAL_DATA actually triggers another data corruption related xftest error (reproducible with passthrough_hp) and I didn't have time to follow up that chain of issues
- got stalled for now, but I need to resume it (unless someone else wants to work on it).
Thanks,
Bernd
Miklos Szeredi
maintainerQuoted 2 lines · Horst Birthelmer
Where do you see the biggest challenge for this. To me, it doesn't look impossible to add that, but I'm sure I'm missing a lot in this context.
The biggest challenge I see is API design. I'd really like to have a simple and consistent API for this.
First task: need to document the current interface, which is long overdue. Here's a start:
https://docs.google.com/document/d/1SInG6nc5dF-db3WtCqFOnRDCY1jRl5JQ0Toiryn2PRc/edit?usp=sharing
This needs refinement, i.e. try to document by cache type, how is the cache used, invalidated, etc.
Thanks,
Miklos
Quoted 7 lines · Miklos Szeredi
Quoted 2 lines · Horst Birthelmer
Where do you see the biggest challenge for this. To me, it doesn't look impossible to add that, but I'm sure I'm missing a lot in this context.
The biggest challenge I see is API design. I'd really like to have a simple and consistent API for this.
OK, that is a valid goal, I guess.
I have tried to come up with a fusex 'module' that does all operations over io-uring since it is a lot easier to just support one way of transport.
Then I have written a very primitive passthrough to test the user space interface and I can get it to work with some small changes to the interface mostly due to the small header space on the ring.
I have done this to check how fast I can have something workable with the new interface since I'm supposed to add compound commands.
Do you think it is worth pursuing further?
Quoted 11 lines
First task: need to document the current interface, which is long overdue. Here's a start:
https://docs.google.com/document/d/1SInG6nc5dF-db3WtCqFOnRDCY1jRl5JQ0Toiryn2PRc/edit?usp=sharing
This needs refinement, i.e. try to document by cache type, how is the cache used, invalidated, etc.
Thanks,
Miklos
Quoted 190 lines · Miklos Szeredi
This stands for "fuse extended/experimental".
The purpose is to provide a clean base for big features like the
FUSE_IOMAPapi.It's also a good way to try new stuff like file handles and compound requests without the risk of breaking something in the large and complex fuse codebase.
Whether these features will be migrated back into the main fuse codebase, or fusex is going to end up as a major version update is still up in the air.
Major differences from regular fuse:
- local filesystem mode only
- only synchronousFUSE_INITis supported
- only no-open mode
- new requests:
+FUSE_LOOKUP_ROOT- return nodeid of root
+FUSE_LOOKUPX-FUSE_LOOKUPwithout the getattr
+FUSE_MKOBJX- mergedFUSE_MKNOD, MKDIR, SYMLINK and TMPFILE
+FUSE_SETSTATX- extended version ofFUSE_SETATTRMissing features:
- file handles / export ops
- compound requests
- xattr caching
- fileattr
- fiemap
- ioctl
-copy_file_range
- lazy dir openTest server can be found at:
https://github.com/szmi/fuse-utils
Signed-off-by Miklos Szeredi <[email protected]>---
Patch is against
git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git#for-nextfs/fuse/Makefile | 2 +- fs/fuse/args.h | 1 + fs/fuse/dev.c | 13 +- fs/fuse/dir.c | 4 +- fs/fuse/fuse_i.h | 8 + fs/fuse/fusex.c | 1751 +++++++++++++++++++++++++++++++++++++ fs/fuse/fusex.h | 4 + fs/fuse/inode.c | 16 +- include/uapi/linux/fuse.h | 36 + 9 files changed, 1826 insertions(+), 9 deletions(-) create mode 100644 fs/fuse/fusex.c create mode 100644 fs/fuse/fusex.hfs/fuse/Makefile +1
-1@@ -12,7 +12,7 @@ obj-$(CONFIG_VIRTIO_FS) += virtiofs.o fuse-y := trace.o # put trace.o first so we see ftrace errors sooner fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o req_timeout.o req.o -fuse-y += poll.o notify.o +fuse-y += poll.o notify.o fusex.o fuse-y += iomode.o fuse-$(CONFIG_FUSE_DAX) += dax.o fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o backing.ofs/fuse/args.h +1
-0@@ -35,6 +35,7 @@ struct fuse_args { bool out_pages:1; bool user_pages:1; bool out_argvar:1; + bool out_var_alloc:1; bool page_zeroing:1; bool page_replace:1; bool may_block:1;fs/fuse/dev.c +11
-2@@ -1847,15 +1847,24 @@ int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args, reqsize +=fuse_len_args (args->out_numargs, args->out_args); -if (reqsize < nbytes || (reqsize > nbytes && !args->out_argvar)) +if (reqsize < nbytes)return -EINVAL ; -else if (reqsize > nbytes) { + +if (args->out_argvar) {struct fuse_arg *lastarg = &args->out_args[args->out_numargs-1 ];unsigned diffsize = reqsize - nbytes;if (diffsize > lastarg->size)return -EINVAL ; lastarg->size -= diffsize; + +if (args->out_var_alloc) { + lastarg->value =kvmalloc (lastarg->size,GFP_KERNEL ); +if (!lastarg->value) +return -ENOMEM ; + } + }else if (reqsize > nbytes) { +return -EINVAL ; }return fuse_copy_args (cs, args->out_numargs, args->out_pages, args->out_args, args->page_zeroing);fs/fuse/dir.c +2
-2@@ -542,7 +542,7 @@ int fuse_valid_type(int m)S_ISBLK (m) ||S_ISFIFO (m) ||S_ISSOCK (m); } -static bool fuse_valid_size (u64 size) +bool fuse_valid_size (u64 size) {return size <=LLONG_MAX ; }@@ -2485,7 +2485,7 @@ static int fuse_symlink_read_folio(struct file *null, struct folio *folio)return err; } -static const struct address_space_operations fuse_symlink_aops = { +const struct address_space_operations fuse_symlink_aops = { .read_folio = fuse_symlink_read_folio, };fs/fuse/fuse_i.h +8
-0@@ -69,6 +69,9 @@ extern struct mutex fuse_mutex;extern unsigned int max_user_bgreq;extern unsigned int max_user_congthresh; +extern struct kmem_cache *fuse_inode_cachep; +extern const struct address_space_operations fuse_symlink_aops; +struct fuse_forget_link ;/** @@ -911,6 +914,8 @@ struct inode *fuse_iget(struct super_block *sb, u64 nodeid,int fuse_lookup_name (struct super_block *sb,u64 nodeid,const struct qstr *name,struct fuse_entry_out *outarg,struct inode **inode); +void fuse_umount_begin (struct super_block *sb); +/* * Initialize READ or READDIR request */ @@ -1102,6 +1107,7 @@ void fuse_ctl_remove_conn(struct fuse_conn *fc);* Is file type valid? */ int fuse_valid_type (int m); +bool fuse_valid_size (u64 size);bool fuse_invalid_attr (struct fuse_attr *attr);@@ -1204,6 +1210,8 @@ struct posix_acl *fuse_get_acl(struct mnt_idmap *idmap,int fuse_set_acl (struct mnt_idmap *,struct dentry *dentry,struct posix_acl *acl,int type); +void fuse_convert_statfs (struct kstatfs *stbuf,struct fuse_kstatfs *attr); +/* readdir.c */ int fuse_readdir (struct file *file,struct dir_context *ctx);fs/fuse/fusex.c new file +7
-0@@ -0,0 +1,1751 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include "fusex.h" +#include "dev.h" +#include "fuse_i.h" + +#include <linux/fs_context.h>
fs/fuse/fusex.c line 17
Quoted 10 lines
fs/fuse/fusex.c new file +10
-0+#include <linux/miscdevice.h> +#include <linux/xxhash.h> +#include <linux/pagemap.h> +#include <linux/exportfs.h> +#include <linux/iversion.h> +#include <linux/posix_acl_xattr.h> +#include <linux/statfs.h> +#include <linux/falloc.h> +#include <linux/fs_parser.h> +
I think you're missing
#include <uapi/linux/magic.h>
here for FUSE_SUPER_MAGIC
Miklos Szeredi
maintainerQuoted 3 lines · Horst Birthelmer
I think you're missing
#include <uapi/linux/magic.h>
here forFUSE_SUPER_MAGIC
Okay.
Thanks,
Miklos
Quoted 35 lines · Miklos Szeredi
This stands for "fuse extended/experimental".
The purpose is to provide a clean base for big features like the
FUSE_IOMAPapi.It's also a good way to try new stuff like file handles and compound requests without the risk of breaking something in the large and complex fuse codebase.
Whether these features will be migrated back into the main fuse codebase, or fusex is going to end up as a major version update is still up in the air.
Major differences from regular fuse:
- local filesystem mode only
- only synchronousFUSE_INITis supported
- only no-open mode
- new requests:
+FUSE_LOOKUP_ROOT- return nodeid of root
+FUSE_LOOKUPX-FUSE_LOOKUPwithout the getattr
+FUSE_MKOBJX- mergedFUSE_MKNOD, MKDIR, SYMLINK and TMPFILE
+FUSE_SETSTATX- extended version ofFUSE_SETATTRMissing features:
- file handles / export ops
- compound requests
- xattr caching
- fileattr
- fiemap
- ioctl
-copy_file_range
- lazy dir open
Quoted 7 lines
Test server can be found at:
https://github.com/szmi/fuse-utils
Signed-off-by Miklos Szeredi <[email protected]>---
Hi Miklos,
I have patched passthrough_hp to use fusex and have noticed, that I had to do the following to make it work:
fs/fuse/fusex.c +2 -0
@@ -14,6 +14,7 @@#include <linux/statfs.h> #include <linux/falloc.h> #include <linux/fs_parser.h> +#include <uapi/linux/magic.h> static void fusex_init_inode (struct inode *inode);
@@ -1337,6 +1338,7 @@ static int fusex_send_opendir(struct fuse_file *ff, struct inode *inode)if (!err) { ff->fh = outarg.fh; ff->open_flags =FOPEN_CACHE_DIR ; + ff->nodeid =get_node_id (inode); }return err; }
---
After that I could actually mount and do simple meta operations. The nodeid is used for listing.
Thanks,
Horst
Miklos Szeredi
maintainerReview static int fusex_send_opendir(struct fuse_file *ff, struct inode *inode) · line 1341
Quoted 5 lines · Horst Birthelmer
(quoted diff) +1
-0@@ -1337,6 +1338,7 @@ static int fusex_send_opendir(struct fuse_file *ff, struct inode *inode) if (!err) { ff->fh = outarg.fh; ff->open_flags = FOPEN_CACHE_DIR; + ff->nodeid = get_node_id(inode);
How about this (untested):
fs/fuse/file.c +0 -0
@@ -594,7 +594,7 @@ void fuse_read_args_fill(struct fuse_io_args *ia,
struct file *file, loff_t pos,
ia->read.in.size = count;
ia->read.in.flags = file->f_flags;
args->opcode = opcode;
- args->nodeid = ff->nodeid;
+ args->nodeid = get_node_id(file_inode(file));
args->in_numargs = 1;
args->in_args[0].size = sizeof(ia->read.in);
args->in_args[0].value = &ia->read.in;fs/fuse/fusex.c +0 -0
@@ -1379,6 +1380,7 @@ static int fusex_dir_release(struct inode
*inode, struct file *file)
ra->inarg.fh = ff->fh;
ra->args.opcode = FUSE_RELEASEDIR;
+ ra->args.nodeid = get_node_id(inode);
ra->args.force = true;
ra->args.nocreds = true;
ra->args.end = fusex_release_end;Thanks,
Miklos
fs/fuse/file.c +1 -1
@@ -594,7 +594,7 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, ia->read.in.size = count; ia->read.in.flags = file->f_flags; args->opcode = opcode; - args->nodeid = ff->nodeid; + args->nodeid =get_node_id (file_inode (file)); args->in_numargs =1 ; args->in_args[0 ].size =sizeof (ia->read.in); args->in_args[0 ].value = &ia->read.in;
fs/fuse/fusex.c +1 -0
@@ -1379,6 +1380,7 @@ static int fusex_dir_release(struct inode *inode, struct file *file) ra->inarg.fh = ff->fh; ra->args.opcode =FUSE_RELEASEDIR ; + ra->args.nodeid =get_node_id (inode); ra->args.force = true; ra->args.nocreds = true; ra->args.end = fusex_release_end;
Quoted 6 lines · Miklos Szeredi
This stands for "fuse extended/experimental".
The purpose is to provide a clean base for big features like the
FUSE_IOMAPapi.
Quoted 7 lines
It's also a good way to try new stuff like file handles and compound requests without the risk of breaking something in the large and complex fuse codebase.
Whether these features will be migrated back into the main fuse codebase, or fusex is going to end up as a major version update is still up in the air.
Could you explain a bit what the main motivation behind fusex is? Is it mainly to clean up interfaces that are now clunky/stale? Or is it more driven by limitations in the current codebase that make newer features hard to add? From the paragraphs above, it kind of seems like the latter?
If it's mostly to clean up interfaces that are now clunky/stale, I wonder if now is the best time given that, as Amir phrased it in [1], "FUSE is experiencing a renaissance of new features and protocol enhancements", and I'm not sure if we know yet if these new interfaces will also have some things we'll have wished in hindsight we had done differently, especially since some of these new features are still actively evolving and gaining new capabilities (including io-uring and passthrough). Would it make sense to let these new features and protocol enhancements evolve and stabilize first before baking them into fusex?
There's a good chance I'm missing something here, but if fusex is mostly to make new features easier to add, I'm not sure I see a lot of difference between doing it in fuse vs fusex when fusex is expanded to have feature-parity with legacy fuse. Is there something in particular that fusex makes easier?
I wonder if libfuse could serve as a bridge for the gap between legacy fuse and fusex, eg libfuse filling any missing gaps / translating between legacy fuse protocols and fusex protocols such that any fuse server written for legacy fuse could run using fusex, which would remove the need to have to do any "backporting" to legacy fuse for new features that are added to fusex and could accelerate the deprecation timeline of legacy fuse.
Quoted 5 lines
Major differences from regular fuse:
- local filesystem mode only
- only synchronousFUSE_INITis supported
Some other things that might be nice:
* handle_killpriv / handle_killpriv2 - sgid/suid + file capabilities clearing will always be handled by the server (default implementation in libfuse)
* in fuse io-uring: headers and payload passed together as one contiguous buffer during registration for pinning, deprecate passing separate buffers per SQE
* in passthrough: backing_fd passed at lookup/creation time only (not at open time), expanded struct fuse_backing_map
Quoted 4 lines
- only no-open mode
- new requests:
+FUSE_LOOKUP_ROOT- return nodeid of root
+FUSE_LOOKUPX-FUSE_LOOKUPwithout the getattr
Could you explain why it's preferred to have lookup disentangled from attributes, eg why compounding lookup + attribute fetching is preferred over embedding the statx directly inside the lookup request without compounding? it was brought up in Luis's lookup handles patchset [2] but it's still a bit unclear to me why compounding the two is better. I think every operation/request that returns a struct fuse_entry_out uses the attributes immediately after through fuse_iget(..., &outarg->attr, ...) or fuse_change_attributes(..., &outarg.attr, ...), so the attributes seemed pretty tightly intercoupled with lookup from what I could see.
Thanks,
Joanne
[1] https://lore.kernel.org/linux-fsdevel/CAOQ4uxjRgQGxWYWbRbjVZy19oA5qKZSd9eANKyD8yCoBNGPNNw@mail.gmail.com/
[2] https://lore.kernel.org/linux-fsdevel/[email protected]/T/#t
Quoted 38 lines
+
FUSE_MKOBJX- mergedFUSE_MKNOD, MKDIR, SYMLINK and TMPFILE
+FUSE_SETSTATX- extended version ofFUSE_SETATTRMissing features:
- file handles / export ops
- compound requests
- xattr caching
- fileattr
- fiemap
- ioctl
-copy_file_range
- lazy dir openTest server can be found at:
https://github.com/szmi/fuse-utils
Signed-off-by Miklos Szeredi <[email protected]>---
Patch is against
git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git#for-nextfs/fuse/Makefile | 2 +- fs/fuse/args.h | 1 + fs/fuse/dev.c | 13 +- fs/fuse/dir.c | 4 +- fs/fuse/fuse_i.h | 8 + fs/fuse/fusex.c | 1751 +++++++++++++++++++++++++++++++++++++ fs/fuse/fusex.h | 4 + fs/fuse/inode.c | 16 +- include/uapi/linux/fuse.h | 36 + 9 files changed, 1826 insertions(+), 9 deletions(-) create mode 100644 fs/fuse/fusex.c create mode 100644 fs/fuse/fusex.h
Miklos Szeredi
maintainerQuoted 5 lines · Joanne Koong
Could you explain a bit what the main motivation behind fusex is? Is it mainly to clean up interfaces that are now clunky/stale? Or is it more driven by limitations in the current codebase that make newer features hard to add? From the paragraphs above, it kind of seems like the latter?
I think the answer is: both.
Adding new features also hard because the old interface is convoluted and inconsistent.
Quoted 10 lines
If it's mostly to clean up interfaces that are now clunky/stale, I wonder if now is the best time given that, as Amir phrased it in [1], "FUSE is experiencing a renaissance of new features and protocol enhancements", and I'm not sure if we know yet if these new interfaces will also have some things we'll have wished in hindsight we had done differently, especially since some of these new features are still actively evolving and gaining new capabilities (including io-uring and passthrough). Would it make sense to let these new features and protocol enhancements evolve and stabilize first before baking them into fusex?
Excellent point.
Currently fusex concerns only the filesystem layer, and is agnostic to the transport layer (except maybe the SYNC_INIT thing). That doesn't mean that we shouldn't have the same API cleanup in the transport layer as well. For example, I'd love to get rid of splice on /dev/fuse, which I feel is more burden than blessing. And I guess fuse-uring could also do with an API cleanup.
As to when and how this should be done? I guess that's up to you and Bernd to decide.
But I feel that we do need to have more high level discussion of APIs, if we want to avoid repeating past mistakes.
Quoted 7 lines
I wonder if libfuse could serve as a bridge for the gap between legacy fuse and fusex, eg libfuse filling any missing gaps / translating between legacy fuse protocols and fusex protocols such that any fuse server written for legacy fuse could run using fusex, which would remove the need to have to do any "backporting" to legacy fuse for new features that are added to fusex and could accelerate the deprecation timeline of legacy fuse.
Definitely. However libfuse does export a lot of details from the kernel API (e.g. INIT flags) that would no longer work with fusex and not really feasible to emulate with libfuse. So some porting would be inevitable.
Quoted 4 lines
Some other things that might be nice:
*handle_killpriv/handle_killpriv2- sgid/suid + file capabilities clearing will always be handled by the server (default implementation in libfuse)
Which is in total contrast to what fusex now does (all that handled by the VFS). Need to resolve this sanely somehow.
Quoted 3 lines
* in fuse io-uring: headers and payload passed together as one contiguous buffer during registration for pinning, deprecate passing separate buffers per SQE
Okay.
Quoted 2 lines
* in passthrough:
backing_fdpassed at lookup/creation time only (not at open time), expanded structfuse_backing_map
Yes, this needs discussion. Amir may have additional info on this.
Quoted 6 lines
Quoted 5 lines
- only no-open mode
- new requests:
+FUSE_LOOKUP_ROOT- return nodeid of root
+FUSE_LOOKUPX-FUSE_LOOKUPwithout the getattr
Quoted 9 lines
Could you explain why it's preferred to have lookup disentangled from attributes, eg why compounding lookup + attribute fetching is preferred over embedding the statx directly inside the lookup request without compounding? it was brought up in Luis's lookup handles patchset [2] but it's still a bit unclear to me why compounding the two is better. I think every operation/request that returns a struct
fuse_entry_outuses the attributes immediately after throughfuse_iget(..., &outarg->attr, ...)orfuse_change_attributes(..., &outarg.attr, ...),
That's one argument for decoupling: why include the same fields (attributes, validity) in several ops (LOOKUP_ROOT, LOOKUPX, MKOBJX) if we already have a separate one to do that? It simplifies the protocol. If we started with decoupled GETATTR, then we wouldn't be taking about adding these X versions, because it would be a simple GETATTR -> STATX conversion.
There's also an interesting side effect of coupling STATX with LOOKUP, which is inherently racy:
Thread A:
send LOOKUP + STATX for /fuse/dir1/filea
[server performs lookup and statx, sends reply]
Thread B:
send SETATTR for /fuse/dir2/fileb, which is hard linked to /fuse/dir1/filea evict inode for /fuse/dir2/fileb
Thread A:
looks up inode for /fuse/dir1/filea in cache, doesn't find any creates inode, fills stale attributes
This issue is currently solved with fc->evict_ctr in regular fuse, but works fine without any additional checks in fusex because of the decoupled ops.
Of course, it is probably desirable to compound LOOKUP with STATX despite this, but it gives the implementation more leeway (e.g. on a different OS).
To conclude: separating out STATX from LOOKUPX, etc. makes the protocol:
a) simpler
b) more flexible
Thanks,
Miklos
Quoted 6 lines · Miklos Szeredi
Quoted 3 lines
* in passthrough:
backing_fdpassed at lookup/creation time only (not at open time), expanded structfuse_backing_mapYes, this needs discussion. Amir may have additional info on this.
Having the two different lifecycles (open-to-close, lookup-to-forget) is by design.
This allows the server to potentially move files from passthrough mode to caching mode and vice versa, as long as all fds are closed.
Adding inode passthrough ops does not play well with this mode this is why it requires the lookup-to-forget lifecycle, but I see no reason kill the open-to-close lifecycle as it seems useful for servers that mostly care about I/O passthrough.
w.r.t expanded struct fuse_backing_map, at the moment it does not sound right to me to map a backing id X with more than a single backing file or device.
I think that the maps of fuse_file to multiple fuse_backing is better handled by another layer of mapping (sort of a backing_mapper) which is what famfs and iomap need, but it is just my initial thoughts.
Joanne, if you are a concrete use case for that, let's try to see
how a design to this use case may look like.
Thanks,
Amir.
Quoted 6 lines · Amir Goldstein
Quoted 4 lines · Miklos Szeredi
Quoted 3 lines
* in passthrough:
backing_fdpassed at lookup/creation time only (not at open time), expanded structfuse_backing_map
Quoted 8 lines
Quoted 2 lines
Yes, this needs discussion. Amir may have additional info on this.
Having the two different lifecycles (open-to-close, lookup-to-forget) is by design.
This allows the server to potentially move files from passthrough mode to caching mode and vice versa, as long as all fds are closed.
I'm unsure how useful mode switching is between caching and passthrough because the server has no control over when the client application closes all its fds (eg it can only react but not trigger it). For my internal use case of switching on passthrough, I was envisioning the server handling everything through direct io until passthrough is possible (eg the remote file has been fully downloaded over the network) so that passthrough can be switched on even with the application having open fds.
Quoted 9 lines
Adding inode passthrough ops does not play well with this mode this is why it requires the lookup-to-forget lifecycle, but I see no reason kill the open-to-close lifecycle as it seems useful for servers that mostly care about I/O passthrough.
w.r.t expanded struct
fuse_backing_map, at the moment it does not sound right to me to map a backing id X with more than a single backing file or device.
My thinking with the struct fuse_backing_map expansion was that future ioctls for managing backing files (eg attaching a backing file to a live inode or swapping one backing file for another) might benefit from a shared struct instead of needing to add new structs for each type.
Quoted 4 lines
I think that the maps of
fuse_fileto multiplefuse_backingis better handled by another layer of mapping (sort of abacking_mapper) which is what famfs and iomap need, but it is just my initial thoughts.
I like this idea a lot, it makes sense to me.
Quoted 3 lines
Joanne, if you are a concrete use case for that, let's try to see
how a design to this use case may look like.
The internal team I'm working with is planning to start working on their server in H2. They will run some benchmarks to see if current passthrough and/or io-uring+zero-copy might get them the performance gains they need, or whether they will still need the set of advanced passthrough features they requested. I'll keep you posted once there's performance data and if there is still a need for the more advanced passthrough features (file mapping to multiple backing files, backing files dynamically changing, enabling passthrough on a live inode), I'll draft up a design doc to get your thoughts on. In the meantime, I'm planning to send out v2 of the inode metadata passthrough patchset sometime this week or next with your feedback from v1.
Thanks,
Joanne
Quoted 3 lines
Thanks,
Amir.
Quoted 14 lines · Joanne Koong
Quoted 12 lines · Amir Goldstein
Quoted 6 lines · Miklos Szeredi
Quoted 3 lines
* in passthrough:
backing_fdpassed at lookup/creation time only (not at open time), expanded structfuse_backing_mapYes, this needs discussion. Amir may have additional info on this.
Having the two different lifecycles (open-to-close, lookup-to-forget) is by design.
Quoted 7 lines
Quoted 2 lines
This allows the server to potentially move files from passthrough mode to caching mode and vice versa, as long as all fds are closed.
I'm unsure how useful mode switching is between caching and passthrough because the server has no control over when the client application closes all its fds (eg it can only react but not trigger it).
This is debatable.
My passthrough fs server coordinates state changes with HSM using flock on open files.
Previous implementations used to allow using cached mode until the file content is fully downloaded, which could be beneficial when streaming large media files.
Current implementation uses the DIO or PASSTHROUGH modes. I have not yet collected enough real world stats to know if this is a problem for large media files streaming, so would prefer to keep the options open for now and I see no reason to deprecate the open-to-close lifecycle. I don't think is adds that much complexity to the code (?).
Quoted 6 lines
For my internal use case of switching on passthrough, I was envisioning the server handling everything through direct io until passthrough is possible (eg the remote file has been fully downloaded over the network) so that passthrough can be switched on even with the application having open fds.
Indeed, being able to switch to passthrough for new fds is a big win, which is why I also changed to this method in the current implementation.
Quoted 27 lines
Quoted 9 lines
Adding inode passthrough ops does not play well with this mode this is why it requires the lookup-to-forget lifecycle, but I see no reason kill the open-to-close lifecycle as it seems useful for servers that mostly care about I/O passthrough.
w.r.t expanded struct
fuse_backing_map, at the moment it does not sound right to me to map a backing id X with more than a single backing file or device.My thinking with the struct
fuse_backing_mapexpansion was that future ioctls for managing backing files (eg attaching a backing file to a live inode or swapping one backing file for another) might benefit from a shared struct instead of needing to add new structs for each type.Quoted 4 lines
I think that the maps of
fuse_fileto multiplefuse_backingis better handled by another layer of mapping (sort of abacking_mapper) which is what famfs and iomap need, but it is just my initial thoughts.I like this idea a lot, it makes sense to me.
Quoted 3 lines
Joanne, if you are a concrete use case for that, let's try to see
how a design to this use case may look like.
Quoted 9 lines
The internal team I'm working with is planning to start working on their server in H2. They will run some benchmarks to see if current passthrough and/or io-uring+zero-copy might get them the performance gains they need, or whether they will still need the set of advanced passthrough features they requested. I'll keep you posted once there's performance data and if there is still a need for the more advanced passthrough features (file mapping to multiple backing files, backing files dynamically changing, enabling passthrough on a live inode), I'll draft up a design doc to get your thoughts on.
Sounds good.
FYI, some of those issues we try to solve at application layer. Our HSM application always tries to keep the inode itself across state changes.
For example, by using clone_file_range() to fill in content. With xfs, we could also try to use atomic swap extents, if file content can change, instead of changing inode, but we did not try this yet.
Quoted 4 lines
In the meantime,
I'm planning to send out v2 of the inode metadata passthrough patchset sometime this week or next with your feedback from v1.
Cool. Let's get this rolling.
I need to get my test setup back in shape.
There were a bunch of fstest failing last time I tested passthrough, mostly related to metadata cache coherency.
I was hoping that GETATTR passthrough could solve some of them. If you have seen any improvements (or regressions) in fstests please share this info with the patch set.
Thanks,
Amir.
Quoted 86 lines · Amir Goldstein
Quoted 14 lines · Joanne Koong
Quoted 12 lines · Amir Goldstein
Quoted 6 lines · Miklos Szeredi
Quoted 3 lines
* in passthrough:
backing_fdpassed at lookup/creation time only (not at open time), expanded structfuse_backing_mapYes, this needs discussion. Amir may have additional info on this.
Having the two different lifecycles (open-to-close, lookup-to-forget) is by design.
Quoted 7 lines
Quoted 2 lines
This allows the server to potentially move files from passthrough mode to caching mode and vice versa, as long as all fds are closed.
I'm unsure how useful mode switching is between caching and passthrough because the server has no control over when the client application closes all its fds (eg it can only react but not trigger it).
This is debatable.
My passthrough fs server coordinates state changes with HSM using flock on open files.Previous implementations used to allow using cached mode until the file content is fully downloaded, which could be beneficial when streaming large media files.
Current implementation uses the DIO or PASSTHROUGH modes. I have not yet collected enough real world stats to know if this is a problem for large media files streaming, so would prefer to keep the options open for now and I see no reason to deprecate the open-to-close lifecycle. I don't think is adds that much complexity to the code (?).
Quoted 6 lines
For my internal use case of switching on passthrough, I was envisioning the server handling everything through direct io until passthrough is possible (eg the remote file has been fully downloaded over the network) so that passthrough can be switched on even with the application having open fds.
Indeed, being able to switch to passthrough for new fds is a big win, which is why I also changed to this method in the current implementation.
Quoted 27 lines
Quoted 9 lines
Adding inode passthrough ops does not play well with this mode this is why it requires the lookup-to-forget lifecycle, but I see no reason kill the open-to-close lifecycle as it seems useful for servers that mostly care about I/O passthrough.
w.r.t expanded struct
fuse_backing_map, at the moment it does not sound right to me to map a backing id X with more than a single backing file or device.My thinking with the struct
fuse_backing_mapexpansion was that future ioctls for managing backing files (eg attaching a backing file to a live inode or swapping one backing file for another) might benefit from a shared struct instead of needing to add new structs for each type.Quoted 4 lines
I think that the maps of
fuse_fileto multiplefuse_backingis better handled by another layer of mapping (sort of abacking_mapper) which is what famfs and iomap need, but it is just my initial thoughts.I like this idea a lot, it makes sense to me.
Quoted 3 lines
Joanne, if you are a concrete use case for that, let's try to see
how a design to this use case may look like.Quoted 9 lines
The internal team I'm working with is planning to start working on their server in H2. They will run some benchmarks to see if current passthrough and/or io-uring+zero-copy might get them the performance gains they need, or whether they will still need the set of advanced passthrough features they requested. I'll keep you posted once there's performance data and if there is still a need for the more advanced passthrough features (file mapping to multiple backing files, backing files dynamically changing, enabling passthrough on a live inode), I'll draft up a design doc to get your thoughts on.
Quoted 7 lines
Sounds good.
FYI, some of those issues we try to solve at application layer. Our HSM application always tries to keep the inode itself across state changes.
For example, by usingclone_file_range()to fill in content. With xfs, we could also try to use atomic swap extents, if file content can change, instead of changing inode, but we did not try this yet.
Thanks for the info. This is useful, I'll look into seeing if some of these requests can be solved at the application layer.
Quoted 8 lines
Quoted 5 lines
In the meantime,
I'm planning to send out v2 of the inode metadata passthrough patchset sometime this week or next with your feedback from v1.Cool. Let's get this rolling.
I need to get my test setup back in shape.
What is the test setup you are using? fstests with passthrough_hp or are there other tests in addition to that? If there's other tests you'd like me to run besides fstests, please let me know and I'd be happy to do that.
When I ran fstests yesterday with passthrough_hp, I saw generic 683, 684, and 685 (dropping suid/sgid on fallocate, fpunch, and fzero) start passing, but beyond that I didn't see any improvements (or regressions).
Thanks,
Joanne
Quoted 8 lines
There were a bunch of fstest failing last time I tested passthrough, mostly related to metadata cache coherency.
I was hoping that GETATTR passthrough could solve some of them. If you have seen any improvements (or regressions) in fstests please share this info with the patch set.Thanks,
Amir.
Quoted 1 line · Joanne Koong
...
Quoted 4 lines
Quoted 2 lines
Cool. Let's get this rolling.
I need to get my test setup back in shape.What is the test setup you are using? fstests with
passthrough_hpor
Yes. *was* using while I developed the passthrough patches. Didn't run it for a while.
Quoted 3 lines
are there other tests in addition to that? If there's other tests you'd like me to run besides fstests, please let me know and I'd be happy to do that.
Nothing beyond the libfuse test_passthrough_hp sanity test and fstests with passthough_hp.
Quoted 5 lines
When I ran fstests yesterday with
passthrough_hp, I saw generic 683, 684, and 685 (dropping suid/sgid on fallocate, fpunch, and fzero) start passing, but beyond that I didn't see any improvements (or regressions).
That's the last report I have on record:
https://lore.kernel.org/linux-fsdevel/CAOQ4uxh=cLySge6htd+DnJrqAKF=C_oJYfVrbpvQGik0wR-+iw@mail.gmail.com/
It speaks of some tests that are
"failing because of missing mtime/atime/ctime updates" and of "failure to strip suid/sgid bits in some cases"
I had imagined that the passthrough of GETATTR would fix some of those failing tests and to some extent, this is in line with your report.
Attribute cache and passthrough is an unfinished business. If we could force passthrough of GETATTR when inode is in passthrough iomode, I wouldn't mind.
Thanks,
Amir.
Quoted 20 lines · Amir Goldstein
Quoted 1 line · Joanne Koong
...
Quoted 4 lines
Quoted 2 lines
Cool. Let's get this rolling.
I need to get my test setup back in shape.What is the test setup you are using? fstests with
passthrough_hporYes. *was* using while I developed the passthrough patches. Didn't run it for a while.
Quoted 3 lines
are there other tests in addition to that? If there's other tests you'd like me to run besides fstests, please let me know and I'd be happy to do that.
Nothing beyond the libfuse
test_passthrough_hpsanity test and fstests withpassthough_hp.Quoted 1 line
Quoted 7 lines
Quoted 4 lines
When I ran fstests yesterday with
passthrough_hp, I saw generic 683, 684, and 685 (dropping suid/sgid on fallocate, fpunch, and fzero) start passing, but beyond that I didn't see any improvements (or regressions).That's the last report I have on record:
https://lore.kernel.org/linux-fsdevel/CAOQ4uxh=cLySge6htd+DnJrqAKF=C_oJYfVrbpvQGik0wR-+iw@mail.gmail.com/
Thanks for the link. I had compared my test results against runs on passthrough_hp --nopassthrough.
Comparing them now to baseline passthrough_hp (no inode passthrough commits) on the tests you listed:
- on baseline passthrough_hp, I'm seeing generic/193 and
generic/355 now pass compared to when you ran them in 2023. generic/080, generic/120, and generic/215 still fail
- with the inode passthrough commits, I see generic/080 and
generic/215 now pass. generic/120 is still failing, which I think is because the backing mount doesn't have noatime but the fuse mount does, so getattr passthrough to the backing file will fetch the updated access time.
Thanks,
Joanne
Quoted 14 lines
It speaks of some tests that are
"failing because of missing mtime/atime/ctime updates" and of "failure to strip suid/sgid bits in some cases"I had imagined that the passthrough of GETATTR would fix some of those failing tests and to some extent, this is in line with your report.
Attribute cache and passthrough is an unfinished business. If we could force passthrough of GETATTR when inode is in passthrough iomode, I wouldn't mind.
Thanks,
Amir.
Quoted 33 lines · Joanne Koong
Quoted 20 lines · Amir Goldstein
Quoted 1 line · Joanne Koong
...
Quoted 4 lines
Quoted 2 lines
Cool. Let's get this rolling.
I need to get my test setup back in shape.What is the test setup you are using? fstests with
passthrough_hporYes. *was* using while I developed the passthrough patches. Didn't run it for a while.
Quoted 3 lines
are there other tests in addition to that? If there's other tests you'd like me to run besides fstests, please let me know and I'd be happy to do that.
Nothing beyond the libfuse
test_passthrough_hpsanity test and fstests withpassthough_hp.Quoted 1 line
Quoted 7 lines
Quoted 4 lines
When I ran fstests yesterday with
passthrough_hp, I saw generic 683, 684, and 685 (dropping suid/sgid on fallocate, fpunch, and fzero) start passing, but beyond that I didn't see any improvements (or regressions).That's the last report I have on record:
https://lore.kernel.org/linux-fsdevel/CAOQ4uxh=cLySge6htd+DnJrqAKF=C_oJYfVrbpvQGik0wR-+iw@mail.gmail.com/Thanks for the link. I had compared my test results against runs on
passthrough_hp--nopassthrough.
Quoted 11 lines
Comparing them now to baseline
passthrough_hp(no inode passthrough commits) on the tests you listed:
- on baselinepassthrough_hp, I'm seeing generic/193 and
generic/355 now pass compared to when you ran them in 2023. generic/080, generic/120, and generic/215 still fail
- with the inode passthrough commits, I see generic/080 and
generic/215 now pass. generic/120 is still failing, which I think is because the backing mount doesn't have noatime but the fuse mount does, so getattr passthrough to the backing file will fetch the updated access time.
Nice! This is what I would have expected.
Thanks,
Amir.
Quoted 14 lines · Miklos Szeredi
Quoted 10 lines · Joanne Koong
If it's mostly to clean up interfaces that are now clunky/stale, I wonder if now is the best time given that, as Amir phrased it in [1], "FUSE is experiencing a renaissance of new features and protocol enhancements", and I'm not sure if we know yet if these new interfaces will also have some things we'll have wished in hindsight we had done differently, especially since some of these new features are still actively evolving and gaining new capabilities (including io-uring and passthrough). Would it make sense to let these new features and protocol enhancements evolve and stabilize first before baking them into fusex?
Quoted 4 lines
Excellent point.
Currently fusex concerns only the filesystem layer, and is agnostic to the transport layer (except maybe the
SYNC_INITthing). That doesn't
As soon as fusex lands, doesn't this mean with the linux backwards compatibility policy that any user-facing transport layer decisions are also locked in? Or is the uapi for it still able to change later because fusex is marked as "experimental"?
Quoted 53 lines
mean that we shouldn't have the same API cleanup in the transport layer as well. For example, I'd love to get rid of splice on
/dev/fuse, which I feel is more burden than blessing. And I guess fuse-uring could also do with an API cleanup.As to when and how this should be done? I guess that's up to you and Bernd to decide.
But I feel that we do need to have more high level discussion of APIs, if we want to avoid repeating past mistakes.
Quoted 6 lines
Quoted 5 lines
- only no-open mode
- new requests:
+FUSE_LOOKUP_ROOT- return nodeid of root
+FUSE_LOOKUPX-FUSE_LOOKUPwithout the getattrQuoted 9 lines
Could you explain why it's preferred to have lookup disentangled from attributes, eg why compounding lookup + attribute fetching is preferred over embedding the statx directly inside the lookup request without compounding? it was brought up in Luis's lookup handles patchset [2] but it's still a bit unclear to me why compounding the two is better. I think every operation/request that returns a struct
fuse_entry_outuses the attributes immediately after throughfuse_iget(..., &outarg->attr, ...)orfuse_change_attributes(..., &outarg.attr, ...),That's one argument for decoupling: why include the same fields (attributes, validity) in several ops (
LOOKUP_ROOT, LOOKUPX, MKOBJX) if we already have a separate one to do that? It simplifies the protocol. If we started with decoupled GETATTR, then we wouldn't be taking about adding these X versions, because it would be a simple GETATTR -> STATX conversion.There's also an interesting side effect of coupling STATX with LOOKUP, which is inherently racy:
Thread A:
send LOOKUP + STATX for /fuse/dir1/filea
[server performs lookup and statx, sends reply]Thread B:
send SETATTR for /fuse/dir2/fileb, which is hard linked to /fuse/dir1/filea evict inode for /fuse/dir2/filebThread A:
looks up inode for /fuse/dir1/filea in cache, doesn't find any creates inode, fills stale attributesThis issue is currently solved with
fc->evict_ctrin regular fuse, but works fine without any additional checks in fusex because of the decoupled ops.
Quoted 8 lines
Of course, it is probably desirable to compound LOOKUP with STATX despite this, but it gives the implementation more leeway (e.g. on a different OS).
To conclude: separating out STATX from LOOKUPX, etc. makes the protocol:
a) simpler
b) more flexible
This makes sense to me, thanks for the explanation.
Thanks,
Joanne
Quoted 17 lines · Joanne Koong
Quoted 16 lines · Miklos Szeredi
Quoted 10 lines · Joanne Koong
If it's mostly to clean up interfaces that are now clunky/stale, I wonder if now is the best time given that, as Amir phrased it in [1], "FUSE is experiencing a renaissance of new features and protocol enhancements", and I'm not sure if we know yet if these new interfaces will also have some things we'll have wished in hindsight we had done differently, especially since some of these new features are still actively evolving and gaining new capabilities (including io-uring and passthrough). Would it make sense to let these new features and protocol enhancements evolve and stabilize first before baking them into fusex?
Excellent point.
Quoted 7 lines
Quoted 2 lines
Currently fusex concerns only the filesystem layer, and is agnostic to the transport layer (except maybe the
SYNC_INITthing). That doesn'tAs soon as fusex lands, doesn't this mean with the linux backwards compatibility policy that any user-facing transport layer decisions are also locked in? Or is the uapi for it still able to change later because fusex is marked as "experimental"?
Create a permissions checking predicate, ala:
bool fuse_iomap_enabled(void)
{
/* Don't let anyone touch iomap until the end of the patchset. */
return false;
/*
* There are fears that a fuse+iomap server could somehow DoS the
* system by doing things like going out to lunch during a writeback
* related iomap request. Only allow iomap access if the fuse server
* has rawio capabilities since those processes can mess things up
* quite well even without our help.
*/
return enable_iomap && has_capability_noaudit(current, CAP_SYS_RAWIO);
}and a default N Kconfig option, and then the only people who can actually use the uabi are fuse developers who are willing to patch and run their own kernels.
--D
Quoted 67 lines
Quoted 53 lines
mean that we shouldn't have the same API cleanup in the transport layer as well. For example, I'd love to get rid of splice on
/dev/fuse, which I feel is more burden than blessing. And I guess fuse-uring could also do with an API cleanup.As to when and how this should be done? I guess that's up to you and Bernd to decide.
But I feel that we do need to have more high level discussion of APIs, if we want to avoid repeating past mistakes.
Quoted 6 lines
Quoted 5 lines
- only no-open mode
- new requests:
+FUSE_LOOKUP_ROOT- return nodeid of root
+FUSE_LOOKUPX-FUSE_LOOKUPwithout the getattrQuoted 9 lines
Could you explain why it's preferred to have lookup disentangled from attributes, eg why compounding lookup + attribute fetching is preferred over embedding the statx directly inside the lookup request without compounding? it was brought up in Luis's lookup handles patchset [2] but it's still a bit unclear to me why compounding the two is better. I think every operation/request that returns a struct
fuse_entry_outuses the attributes immediately after throughfuse_iget(..., &outarg->attr, ...)orfuse_change_attributes(..., &outarg.attr, ...),That's one argument for decoupling: why include the same fields (attributes, validity) in several ops (
LOOKUP_ROOT, LOOKUPX, MKOBJX) if we already have a separate one to do that? It simplifies the protocol. If we started with decoupled GETATTR, then we wouldn't be taking about adding these X versions, because it would be a simple GETATTR -> STATX conversion.There's also an interesting side effect of coupling STATX with LOOKUP, which is inherently racy:
Thread A:
send LOOKUP + STATX for /fuse/dir1/filea
[server performs lookup and statx, sends reply]Thread B:
send SETATTR for /fuse/dir2/fileb, which is hard linked to /fuse/dir1/filea evict inode for /fuse/dir2/filebThread A:
looks up inode for /fuse/dir1/filea in cache, doesn't find any creates inode, fills stale attributesThis issue is currently solved with
fc->evict_ctrin regular fuse, but works fine without any additional checks in fusex because of the decoupled ops.Quoted 8 lines
Of course, it is probably desirable to compound LOOKUP with STATX despite this, but it gives the implementation more leeway (e.g. on a different OS).
To conclude: separating out STATX from LOOKUPX, etc. makes the protocol:
a) simpler
b) more flexibleThis makes sense to me, thanks for the explanation.
Thanks,
Joanne
Quoted 13 lines · Miklos Szeredi
This stands for "fuse extended/experimental".
The purpose is to provide a clean base for big features like the
FUSE_IOMAPapi.It's also a good way to try new stuff like file handles and compound requests without the risk of breaking something in the large and complex fuse codebase.
Whether these features will be migrated back into the main fuse codebase, or fusex is going to end up as a major version update is still up in the air.
[...]
fs/fuse/fusex.c line 27
Quoted 2 lines
fs/fuse/fusex.c new file +2
-0+#define ADD_IN_ARG_ZERO (args) \ + ADD_IN_ARG(args, 0, NULL)
Hi Miklos,
could we avoid that? If we don't want to change existing op codes, could we avoid it at least for any new op code?
DDN currently has a problem with
LOOKUP
STATX
and fuse_permission() calling into getattr.
Issues come from DLM and distributed metadata operations. It is relatively simple to handle a total dentry/inode invalidation when a file gets removed or moved to another dir. The really hard part are time stamps and file size. I.e. most of the time fuse-client attributes are just in state invalid. But then for single client access it also makes sense to have attributes cached for ininity. So DLM invalidates attributes if things change on a remote system.
Issue are now file creations of multiple clients in the same dir - the parent does not have updated file attributes and actually does not need to have them either for plain revalidate or permission attribute fetch.
For perm getattr it is relatively straight forward to use statx and add in sx_flags that the server has to fetch. It can respond with more and then cache time stamp gets updated. If some attr missing, time stamp is not changed (detable it should be set to invalid)
Revalidate is harder, in the range to impossible with the curent FUSE_LOOKUP call. There is no header, where a flag could get passed in that it is a revalidate and not all attribtes are needed.
From my point of view, every single op code should have their own u32 feature flags field, so that an op code is easy to extend and without the need to update the global flags.
[...]
fs/fuse/fusex.c line 659
Quoted 10 lines
fs/fuse/fusex.c new file +10
-0+ +static struct inode *fusex_do_lookup (struct inode *base,const struct qstr *name) +{ +FUSE_ARGS (args); +struct fuse_entryx_out outarg; +struct fusex_id id; +int err; + + args.opcode =FUSE_LOOKUPX ; +ADD_IN_ARG_ZERO (args);
Here is would be good to have something like
#define FUSE_LOOKUPX_IN_IS_REVALIDATE (1u << )
struct fuse_lookupx_in {
int32_t op_flags; /* op code feature flags */
int32_t lookup_flags; /* so far FUSE_LOOKUPX_IN_IS_REVALIDATE */
};
Thanks,
Bernd
Miklos Szeredi
maintainerQuoted 5 lines · Bernd Schubert
fs/fuse/fusex.c line 27
Quoted 2 lines
fs/fuse/fusex.c new file +2
-0+#define ADD_IN_ARG_ZERO (args) \ + ADD_IN_ARG(args, 0, NULL)could we avoid that? If we don't want to change existing op codes, could we avoid it at least for any new op code?
Good point.
What we really want is fixed size headers, and what better time to do that with fusex.
Do you see any issue with just taking the 128 + 128 bytes that fuse-uring is using and padding the headers with zeroes?
fuse_setstatx_in goes over the limit, but we can just move the statx part to the payload.
Thanks,
Miklos
Hi Miklos!
I'm not sure if you have a more up-to-date version of your patch (or patchset, as you mentioned you'd be splitting it). But here's a small issue I have fixed locally a while back, when I started playing with fusex:
On Wed, Apr 29 2026, Miklos Szeredi wrote:
<snip>
fs/fuse/fusex.c line 308
Quoted 11 lines
fs/fuse/fusex.c new file +11
-0+static struct inode *fusex_get_inode (struct super_block *sb,struct fusex_id *id) +{ +struct inode *inode =fusex_iget (sb, id); +struct fuse_statx_out statx; +int err; + +if (!inode) +return ERR_PTR (-ENOMEM ); + +if (inode_state_read_once (inode) &I_NEW ) { +inode_state_set (inode,I_DONTCACHE );
inode_state_set() assumes the caller holds inode->i_lock, so we will see a kernel splat from a lockdep assertion.
Cheers,
Signature · 1 line
Luís
Quoted 11 lines
fs/fuse/fusex.c new file +11
-0+ err =fusex_send_statx (inode, &statx); +if (err) { +discard_new_inode (inode); +return ERR_PTR (err); + } +fusex_set_attr (inode, &statx.stat); +fusex_init_inode (inode); +unlock_new_inode (inode); + } +return inode; +}