erofs: introduce Rust implementation
Subsystems (3)
cover erofs: introduce Rust implementation
Greetings,
So here is a patchset to add Rust skeleton codes to the current EROFS implementation. The implementation is deeply inspired by the current C implementation, and it's based on a generic erofs_sys crate[1] written by me. The purpose is to potentially replace some of C codes to make to make full use of Rust's safety features and better optimization guarantees.
Many of the features (like compression inodes) still
fall back to C implementation because of my limited time and lack of Rust counterparts. However, the Extended Attributes work purely in Rust.
Some changes are done to the original C code.
1) Some of superblock operations are modified slightly to make sure memory allocation and deallocation are done correctly when interacting with Rust.
2) A new rust_helpers.c file is introduced to help Rust to deal with self-included EROFS API without exporting types that are not interpreted in Rust.
3) A new rust_bindings.h is introduced to provide Rust functions externs with the same function signature as Rust side so that C-side code can use the bindings easily.
4) CONFIG_EROFS_FS_RUST is added in dir.c, inode.c, super.c, data.c, and xattr.c to allow C code to be opt out and uses Rust implementation.
5) Some macros and function signatures are tweaked in internal.h with the compilation options.
Note that, since currently there is no mature Rust VFS implementation landed upstream, this patchset only uses C bindings internally and each unsafe operation is examined. This implementation only offers C-ABI-compatible functions impls and gets its exposed to original C implementation as either hooks or function pointers.
Also note that, this patchset only uses already-present self-included EROFS API and it uses as few C bindings generated from bindgen as possible, only inode, dentry, file and dir_context related are used, to be precise.
Since the EROFS community is pretty interested in giving Rust a try, I think this patchset will be a good start for Rust EROFS.
This patchset is based on the latest EROFS development tree. And the current codebase can also be found on my github repo[2].
[1]: https://github.com/ToolmanP/erofs-rs
[2]: https://github.com/ToolmanP/erofs-rs-linux
Yiyang Wu (24):
erofs: lift up erofs_fill_inode to global
erofs: add superblock data structure in Rust
erofs: add Errno in Rust
erofs: add xattrs data structure in Rust
erofs: add inode data structure in Rust
erofs: add alloc_helper in Rust
erofs: add data abstraction in Rust
erofs: add device data structure in Rust
erofs: add continuous iterators in Rust
erofs: add device_infos implementation in Rust
erofs: add map data structure in Rust
erofs: add directory entry data structure in Rust
erofs: add runtime filesystem and inode in Rust
erofs: add block mapping capability in Rust
erofs: add iter methods in filesystem in Rust
erofs: implement dir and inode operations in Rust
erofs: introduce Rust SBI to C
erofs: introduce iget alternative to C
erofs: introduce namei alternative to C
erofs: introduce readdir alternative to C
erofs: introduce erofs_map_blocks alternative to C
erofs: add skippable iters in Rust
erofs: implement xattrs operations in Rust
erofs: introduce xattrs replacement to C
fs/erofs/Kconfig | 10 + fs/erofs/Makefile | 4 + fs/erofs/data.c | 5 + fs/erofs/data_rs.rs | 63 +++ fs/erofs/dir.c | 2 + fs/erofs/dir_rs.rs | 57 ++ fs/erofs/inode.c | 10 +- fs/erofs/inode_rs.rs | 64 +++ fs/erofs/internal.h | 47 ++ fs/erofs/namei.c | 2 + fs/erofs/namei_rs.rs | 56 ++ fs/erofs/rust/erofs_sys.rs | 47 ++ fs/erofs/rust/erofs_sys/alloc_helper.rs | 35 ++ fs/erofs/rust/erofs_sys/data.rs | 70 +++ fs/erofs/rust/erofs_sys/data/backends.rs | 4 + .../erofs_sys/data/backends/uncompressed.rs | 39 ++ fs/erofs/rust/erofs_sys/data/raw_iters.rs | 127 +++++ .../rust/erofs_sys/data/raw_iters/ref_iter.rs | 131 +++++ .../rust/erofs_sys/data/raw_iters/traits.rs | 17 + fs/erofs/rust/erofs_sys/devices.rs | 75 +++ fs/erofs/rust/erofs_sys/dir.rs | 98 ++++ fs/erofs/rust/erofs_sys/errnos.rs | 191 +++++++ fs/erofs/rust/erofs_sys/inode.rs | 398 ++++++++++++++ fs/erofs/rust/erofs_sys/map.rs | 99 ++++ fs/erofs/rust/erofs_sys/operations.rs | 62 +++ fs/erofs/rust/erofs_sys/superblock.rs | 514 ++++++++++++++++++ fs/erofs/rust/erofs_sys/superblock/mem.rs | 94 ++++ fs/erofs/rust/erofs_sys/xattrs.rs | 272 +++++++++ fs/erofs/rust/kinode.rs | 76 +++ fs/erofs/rust/ksources.rs | 66 +++ fs/erofs/rust/ksuperblock.rs | 30 + fs/erofs/rust/mod.rs | 7 + fs/erofs/rust_bindings.h | 39 ++ fs/erofs/rust_helpers.c | 86 +++ fs/erofs/rust_helpers.h | 23 + fs/erofs/super.c | 51 +- fs/erofs/super_rs.rs | 59 ++ fs/erofs/xattr.c | 31 +- fs/erofs/xattr.h | 7 + fs/erofs/xattr_rs.rs | 106 ++++ 40 files changed, 3153 insertions(+), 21 deletions(-) create mode 100644 fs/erofs/data_rs.rs create mode 100644 fs/erofs/dir_rs.rs create mode 100644 fs/erofs/inode_rs.rs create mode 100644 fs/erofs/namei_rs.rs create mode 100644 fs/erofs/rust/erofs_sys.rs create mode 100644 fs/erofs/rust/erofs_sys/alloc_helper.rs create mode 100644 fs/erofs/rust/erofs_sys/data.rs create mode 100644 fs/erofs/rust/erofs_sys/data/backends.rs create mode 100644 fs/erofs/rust/erofs_sys/data/backends/uncompressed.rs create mode 100644 fs/erofs/rust/erofs_sys/data/raw_iters.rs create mode 100644 fs/erofs/rust/erofs_sys/data/raw_iters/ref_iter.rs create mode 100644 fs/erofs/rust/erofs_sys/data/raw_iters/traits.rs create mode 100644 fs/erofs/rust/erofs_sys/devices.rs create mode 100644 fs/erofs/rust/erofs_sys/dir.rs create mode 100644 fs/erofs/rust/erofs_sys/errnos.rs create mode 100644 fs/erofs/rust/erofs_sys/inode.rs create mode 100644 fs/erofs/rust/erofs_sys/map.rs create mode 100644 fs/erofs/rust/erofs_sys/operations.rs create mode 100644 fs/erofs/rust/erofs_sys/superblock.rs create mode 100644 fs/erofs/rust/erofs_sys/superblock/mem.rs create mode 100644 fs/erofs/rust/erofs_sys/xattrs.rs create mode 100644 fs/erofs/rust/kinode.rs create mode 100644 fs/erofs/rust/ksources.rs create mode 100644 fs/erofs/rust/ksuperblock.rs create mode 100644 fs/erofs/rust/mod.rs create mode 100644 fs/erofs/rust_bindings.h create mode 100644 fs/erofs/rust_helpers.c create mode 100644 fs/erofs/rust_helpers.h create mode 100644 fs/erofs/super_rs.rs create mode 100644 fs/erofs/xattr_rs.rs
Signature · 1 line
2.46.0
RFC PATCH 1/24 erofs: lift up erofs_fill_inode to global
Lift up erofs_fill_inode as a global symbol so that rust_helpers can use it for better compatibility.
--- fs/erofs/inode.c | 2 +- fs/erofs/internal.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-)
fs/erofs/inode.c +1 -1
@@ -196,7 +196,7 @@ static int erofs_read_inode(struct inode *inode)return err; } -static int erofs_fill_inode (struct inode *inode) +int erofs_fill_inode (struct inode *inode) {struct erofs_inode *vi =EROFS_I (inode);int err;
fs/erofs/internal.h +1 -0
@@ -416,6 +416,7 @@ int erofs_map_blocks(struct inode *inode, struct erofs_map_blocks *map);void erofs_onlinefolio_init (struct folio *folio);void erofs_onlinefolio_split (struct folio *folio);void erofs_onlinefolio_end (struct folio *folio,int err); +int erofs_fill_inode (struct inode *inode);struct inode *erofs_iget (struct super_block *sb,erofs_nid_t nid); int erofs_getattr(struct mnt_idmap *idmap, const struct path *path, struct kstat *stat, u32 request_mask,
Signature · 1 line
2.46.0
RFC PATCH 2/24 erofs: add superblock data structure in Rust
In reply to Yiyang WuThis patch adds a compilable super_rs.rs and introduces superblock data structure in Rust. Note that this patch leaves C-side code untouched.
--- fs/erofs/Kconfig | 10 ++ fs/erofs/Makefile | 1 + fs/erofs/rust/erofs_sys.rs | 22 +++++ fs/erofs/rust/erofs_sys/superblock.rs | 132 ++++++++++++++++++++++++++ fs/erofs/rust/mod.rs | 4 + fs/erofs/super_rs.rs | 9 ++ 6 files changed, 178 insertions(+) create mode 100644 fs/erofs/rust/erofs_sys.rs create mode 100644 fs/erofs/rust/erofs_sys/superblock.rs create mode 100644 fs/erofs/rust/mod.rs create mode 100644 fs/erofs/super_rs.rs
fs/erofs/Kconfig +10 -0
@@ -178,3 +178,13 @@ config EROFS_FS_PCPU_KTHREAD_HIPRI at higher priority. If unsure, say N. + +config EROFS_FS_RUST + bool "EROFS use RUST Replacement (EXPERIMENTAL)" + depends on EROFS_FS && RUST + help + This permits EROFS to use EXPERIMENTAL Rust implementation + for EROFS. This should be considered as an experimental + feature for now. + + If unsure, say N.
fs/erofs/Makefile +1 -0
@@ -9,3 +9,4 @@ erofs-$(CONFIG_EROFS_FS_ZIP_DEFLATE) += decompressor_deflate.o erofs-$(CONFIG_EROFS_FS_ZIP_ZSTD) += decompressor_zstd.o erofs-$(CONFIG_EROFS_FS_BACKED_BY_FILE) += fileio.o erofs-$(CONFIG_EROFS_FS_ONDEMAND) += fscache.o +erofs-$(CONFIG_EROFS_FS_RUST) += super_rs.o
fs/erofs/rust/erofs_sys.rs new file +22 -0
@@ -0,0 +1,22 @@ +#![allow(dead_code)] +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later + +//! A pure Rust implementation of the EROFS filesystem. +//! Technical Details are documented in the [EROFS Documentation](https://erofs.docs.kernel.org/en/latest/) + +// It's unavoidable to import alloc here. Since there are so many backends there and if we want to +// to use trait object to export Filesystem pointer. The alloc crate here is necessary. + +#[cfg(not( CONFIG_EROFS_FS = "y" ))] +extern crate alloc; + +/// Erofs requires block index to a 32 bit unsigned integer. +pub (crate )type Blk =u32 ; +/// Erofs requires normal offset to be a 64bit unsigned integer. +pub (crate )type Off =u64 ; +/// Erofs requires inode nid to be a 64bit unsigned integer. +pub (crate )type Nid =u64 ; +/// Erofs Super Offset to read the ondisk superblock +pub (crate )const EROFS_SUPER_OFFSET :Off =1024 ; +pub (crate )mod superblock;
fs/erofs/rust/erofs_sys/superblock.rs new file +132 -0
132 lines not on this page · raw message
fs/erofs/rust/mod.rs new file +4 -0
@@ -0,0 +1,4 @@ +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later + +pub (crate )mod erofs_sys;
fs/erofs/super_rs.rs new file +9 -0
@@ -0,0 +1,9 @@ +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later + +//! EROFS Rust Kernel Module Helpers Implementation +//! This is only for experimental purpose. Feedback is always welcome. + +#[allow(dead_code)] +#[allow(missing_docs)] +pub (crate )mod rust;
Signature · 1 line
2.46.0
Re: [RFC PATCH 02/24] erofs: add superblock data structure in Rust
fs/erofs/rust/erofs_sys.rs line 3
Quoted 9 lines · Yiyang Wu
fs/erofs/rust/erofs_sys.rs new file +3
-0@@ -0,0 +1,22 @@ +#![allow(dead_code)] +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later
Sorry, but I have to ask, why a dual license here? You are only linking to GPL-2.0-only code, so why the different license? Especially if you used the GPL-2.0-only code to "translate" from.
If you REALLY REALLY want to use a dual license, please get your lawyers to document why this is needed and put it in the changelog for the next time you submit this series when adding files with dual licenses so I don't have to ask again :)
thanks,
greg k-h
Re: [RFC PATCH 02/24] erofs: add superblock data structure in Rust
Hi Greg,
Quoted 15 lines · Greg KH
fs/erofs/rust/erofs_sys.rs line 3
Quoted 9 lines · Yiyang Wu
fs/erofs/rust/erofs_sys.rs new file +3
-0@@ -0,0 +1,22 @@ +#![allow(dead_code)] +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later Sorry, but I have to ask, why a dual license here? You are only linking to GPL-2.0-only code, so why the different license? Especially if you used the GPL-2.0-only code to "translate" from.
Quoted 4 lines
If you REALLY REALLY want to use a dual license, please get your lawyers to document why this is needed and put it in the changelog for the next time you submit this series when adding files with dual licenses so I don't have to ask again :)
As a new Rust kernel developper, Yiyang is working on EROFS Rust userspace implementation too.
I think he just would like to share the common Rust logic between kernel and userspace. Since for the userspace side, Apache-2.0 or even MIT is more friendly for 3rd applications (especially cloud-native applications). So the dual license is proposed here, if you don't have strong opinion, I will ask Yiyang document this in the next version. Or we're fine to drop MIT too.
Thanks,
Gao Xiang
Quoted 4 lines
thanks,
greg k-h
Re: [RFC PATCH 02/24] erofs: add superblock data structure in Rust
Quoted 23 lines · Gao Xiang
Hi Greg,
Quoted 15 lines · Greg KH
fs/erofs/rust/erofs_sys.rs line 3
Quoted 9 lines · Yiyang Wu
fs/erofs/rust/erofs_sys.rs new file +3
-0@@ -0,0 +1,22 @@ +#![allow(dead_code)] +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later Sorry, but I have to ask, why a dual license here? You are only linking to GPL-2.0-only code, so why the different license? Especially if you used the GPL-2.0-only code to "translate" from.
Quoted 4 lines
If you REALLY REALLY want to use a dual license, please get your lawyers to document why this is needed and put it in the changelog for the next time you submit this series when adding files with dual licenses so I don't have to ask again :)
Quoted 5 lines
As a new Rust kernel developper, Yiyang is working on EROFS Rust userspace implementation too.
I think he just would like to share the common Rust logic between kernel and userspace.
Is that actually possible here? This is very kernel-specific code from what I can tell, and again, it's based on the existing GPL-v2 code, so you are kind of changing the license in the transformation to a different language, right?
Quoted 5 lines
Since for the userspace side, Apache-2.0
or even MIT is more friendly for 3rd applications (especially cloud-native applications). So the dual license is proposed here, if you don't have strong opinion, I will ask Yiyang document this in the next version. Or we're fine to drop MIT too.
If you do not have explicit reasons to do this, AND legal approval with the understanding of how to do dual license kernel code properly, I would not do it at all as it's a lot of extra work. Again, talk to your lawyers about this please. And if you come up with the "we really want to do this," great, just document it properly as to what is going on here and why this decision is made.
thanks,
greg k-h
Re: [RFC PATCH 02/24] erofs: add superblock data structure in Rust
Quoted 27 lines · Greg KH
Quoted 26 lines · Gao Xiang
Hi Greg,
Quoted 15 lines · Greg KH
fs/erofs/rust/erofs_sys.rs line 3
Quoted 9 lines · Yiyang Wu
fs/erofs/rust/erofs_sys.rs new file +3
-0@@ -0,0 +1,22 @@ +#![allow(dead_code)] +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later Sorry, but I have to ask, why a dual license here? You are only linking to GPL-2.0-only code, so why the different license? Especially if you used the GPL-2.0-only code to "translate" from.
Quoted 4 lines
If you REALLY REALLY want to use a dual license, please get your lawyers to document why this is needed and put it in the changelog for the next time you submit this series when adding files with dual licenses so I don't have to ask again :)
As a new Rust kernel developper, Yiyang is working on EROFS Rust userspace implementation too.
Quoted 7 lines
Quoted 2 lines
I think he just would like to share the common Rust logic between kernel and userspace.
Is that actually possible here? This is very kernel-specific code from what I can tell, and again, it's based on the existing GPL-v2 code, so you are kind of changing the license in the transformation to a different language, right?
It's possible, Yiyang implemented a total userspace Rust crates to parse EROFS format with limited APIs:
https://github.com/ToolmanP/erofs-rs
Also take another C example, kernel XFS (fs/libxfs) and xfsprogs (userspace) use the same codebase. Although they both use GPL license only.
Quoted 7 lines
Quoted 6 lines
Since for the userspace side, Apache-2.0
or even MIT is more friendly for 3rd applications (especially cloud-native applications). So the dual license is proposed here, if you don't have strong opinion, I will ask Yiyang document this in the next version. Or we're fine to drop MIT too.
Quoted 6 lines
If you do not have explicit reasons to do this, AND legal approval with the understanding of how to do dual license kernel code properly, I would not do it at all as it's a lot of extra work. Again, talk to your lawyers about this please. And if you come up with the "we really want to do this," great, just document it properly as to what is going on here and why this decision is made.
Ok, then let's stay with GPL only. Although as I mentioned, cloud-native applications are happy with Apache-2.0 or MIT, which means there could be diverged for kernel and userspace on the Rust side too.
Thanks,
Gao Xiang
Quoted 4 lines
thanks,
greg k-h
Re: [RFC PATCH 02/24] erofs: add superblock data structure in Rust
In reply to Greg KHQuoted 15 lines · Greg KH
fs/erofs/rust/erofs_sys.rs line 3
Quoted 9 lines · Yiyang Wu
fs/erofs/rust/erofs_sys.rs new file +3
-0@@ -0,0 +1,22 @@ +#![allow(dead_code)] +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later Sorry, but I have to ask, why a dual license here? You are only linking to GPL-2.0-only code, so why the different license? Especially if you used the GPL-2.0-only code to "translate" from.
Quoted 8 lines
If you REALLY REALLY want to use a dual license, please get your lawyers to document why this is needed and put it in the changelog for the next time you submit this series when adding files with dual licenses so I don't have to ask again :)
thanks,
greg k-h
C'mon, I just don't want this discussion to be heated.
I mean my original code is licensed under MIT and I've already learned that Linux is under GPL-2.0. So i originally thought modifying it to dual licenses can help address incompatiblity issues. According to wikipedia, may I quote: "When software is multi-licensed, recipients can typically choose the terms under which they want to use or distribute the software, but the simple presence of multiple licenses in a software package or library does not necessarily indicate that the recipient can freely choose one or the other."[1], so according to this, I believe putting these under a GPL-2.0 project should be OK, since it will be forcily licensed **only** under GPL-2.0.
Since I wasn't involved in Kernel Development before,
I just don't know you guys attitudes towards this kind of stuff. If you guys are not pretty happy with this, I can just switch back to GPL-2.0 and it's a big business for me.
Best Regards,
Yiyang Wu.
Re: [RFC PATCH 02/24] erofs: add superblock data structure in Rust
In reply to Greg KHQuoted 15 lines · Greg KH
fs/erofs/rust/erofs_sys.rs line 3
Quoted 9 lines · Yiyang Wu
fs/erofs/rust/erofs_sys.rs new file +3
-0@@ -0,0 +1,22 @@ +#![allow(dead_code)] +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later Sorry, but I have to ask, why a dual license here? You are only linking to GPL-2.0-only code, so why the different license? Especially if you used the GPL-2.0-only code to "translate" from.
Quoted 8 lines
If you REALLY REALLY want to use a dual license, please get your lawyers to document why this is needed and put it in the changelog for the next time you submit this series when adding files with dual licenses so I don't have to ask again :)
thanks,
greg k-h
C'Mon, I have no intension to make this discussion look heated.
I mean what I original code is under MIT and i've learned that Linux is GPL-2.0, so I naively thought it's OK to dual licensed this to support flexibility according the Wikipedia, should I quote: "When software is multi-licensed, recipients can typically choose the terms under which they want to use or distribute the software, but the simple presence of multiple licenses in a software package or library does not necessarily indicate that the recipient can
freely choose one or the other. "[1]. Since it says multiple licenses does not necessarily indicate that the recipient can freely choose one or other,I thought the strictest license applies here and it should GPL-2.0-only in this case.
I don't have any previous experience in Kernel Development so I really just have no ideas about you guys attitude towards this kind of issue. If insisted on switching back to GPL-2.0-only code, It's fine for me and i'llchange this in the next version. Again I don't have this kind of knowledge in advance, and if multi-license is inspected case-by-case, project-by-project, then I will take notes and never make this kind of mistakes again.
Best Regards,
Yiyang Wu.
RFC PATCH 3/24 erofs: add Errno in Rust
In reply to Yiyang WuIntroduce Errno to Rust side code. Note that in current Rust For Linux, Errnos are implemented as core::ffi::c_uint unit structs. However, EUCLEAN, a.k.a EFSCORRUPTED is missing from error crate.
Since the errno_base hasn't changed for over 13 years, This patch merely serves as a temporary workaround for the missing errno in the Rust For Linux.
--- fs/erofs/rust/erofs_sys.rs | 6 + fs/erofs/rust/erofs_sys/errnos.rs | 191 ++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 fs/erofs/rust/erofs_sys/errnos.rs
fs/erofs/rust/erofs_sys.rs +6 -0
@@ -19,4 +19,10 @@pub (crate )type Nid =u64 ;/// Erofs Super Offset to read the ondisk superblock pub (crate )const EROFS_SUPER_OFFSET :Off =1024 ; +/// PosixResult as a type alias to kernel::error::Result +/// to avoid naming conflicts. +pub (crate )type PosixResult <T > =Result <T ,Errno >; + +pub (crate )mod errnos;pub (crate )mod superblock; +pub (crate )use errnos::Errno ;
fs/erofs/rust/erofs_sys/errnos.rs new file +191 -0
191 lines not on this page · raw message
Signature · 1 line
2.46.0
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Quoted 7 lines · Yiyang Wu
Introduce Errno to Rust side code. Note that in current Rust For Linux, Errnos are implemented as
core::ffi::c_uintunit structs. However, EUCLEAN, a.k.a EFSCORRUPTED is missing from error crate.Since the
errno_basehasn't changed for over 13 years, This patch merely serves as a temporary workaround for the missing errno in the Rust For Linux.
Why not just add the missing errno to the core rust code instead? No need to define a whole new one for this.
thanks,
greg k-h
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Hi Greg,
Quoted 11 lines · Greg KH
Quoted 7 lines · Yiyang Wu
Introduce Errno to Rust side code. Note that in current Rust For Linux, Errnos are implemented as
core::ffi::c_uintunit structs. However, EUCLEAN, a.k.a EFSCORRUPTED is missing from error crate.Since the
errno_basehasn't changed for over 13 years, This patch merely serves as a temporary workaround for the missing errno in the Rust For Linux.Why not just add the missing errno to the core rust code instead? No need to define a whole new one for this.
I've discussed with Yiyang about this last week. I also tend to avoid our own errno.
The main reason is that Rust errno misses EUCLEAN error number. TBH, I don't know why not just introduces all kernel supported errnos for Rust in one shot.
I guess just because no Rust user uses other errno? But for errno cases, I think it's odd for users to add their own errno.
Thanks,
Gao Xiang
Quoted 4 lines
thanks,
greg k-h
cover rust: introduce declare_err! autogeneration
In reply to Greg KHCurrently, the error.rs's errno import is done manually by copying the comments from include/linux/errno.h and uses the declare_err! to wrap the constant errno.
However, this reduces the readability and increases difficulty of maintaining the error.rs if the errno list is growing too long or or if errno.h gets updated for new semantics.
This patchset solves this issues for good by introducing a rule to generate errno_generated.rs by seding the errno.h and including the generated file in the error.rs.
This patch is based on the rust-next branch.
Yiyang Wu (1):
rust: error: auto-generate error declarations
rust/.gitignore | 1 + rust/Makefile | 14 ++++++++++- rust/kernel/error.rs | 58 +++----------------------------------------- 3 files changed, 18 insertions(+), 55 deletions(-)
Signature · 1 line
2.46.0
PATCH RESEND 1/1 rust: error: auto-generate error declarations
This patch adds a new cmd_errno to convert the include/linux/errno.h content into declare_err! macros for better maintainability and readability.
--- rust/.gitignore | 1 + rust/Makefile | 14 ++++++++++- rust/kernel/error.rs | 58 +++----------------------------------------- 3 files changed, 18 insertions(+), 55 deletions(-)
rust/.gitignore +1 -0
@@ -5,6 +5,7 @@ bindings_helpers_generated.rs doctests_kernel_generated.rs doctests_kernel_generated_kunit.c uapi_generated.rs +errno_generated.rs exports_*_generated.h doc/ test/
rust/Makefile +13 -1
@@ -22,6 +22,8 @@ always-$(CONFIG_RUST) += exports_alloc_generated.h exports_helpers_generated.h \ always-$(CONFIG_RUST) += uapi/uapi_generated.rs obj-$(CONFIG_RUST) += uapi.o +always-$(CONFIG_RUST) += kernel/errno_generated.rs + ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW obj-$(CONFIG_RUST) += build_error.o else
@@ -289,6 +291,15 @@ $(obj)/uapi/uapi_generated.rs: $(src)/uapi/uapi_helper.h \ $(src)/bindgen_parameters FORCE $(call if_changed_dep,bindgen) +quiet_cmd_errno = EXPORTS $@ + cmd_errno = \ + $(CC) $(c_flags) -E -CC -dD $< \ + | sed -E 's/\#define\s*([A-Z0-9]+)\s+([0-9]+)\s+\/\*\s*(.*)\s\*\//declare_err!(\1, "\3.");/' \ + | grep -E '^declare_err.*$$' > $@ + +$(obj)/kernel/errno_generated.rs: $(srctree)/include/linux/errno.h FORCE + $(call if_changed,errno) + # See `CFLAGS_REMOVE_helpers.o` above. In addition, Clang on C does not warn # with `-Wmissing-declarations` (unlike GCC), so it is not strictly needed here # given it is `libclang`; but for consistency, future Clang changes and/or
@@ -420,7 +431,8 @@ $(obj)/uapi.o: $(src)/uapi/lib.rs \ $(obj)/kernel.o: private rustc_target_flags = --extern alloc \ --extern build_error --extern macros --extern bindings --extern uapi -$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/alloc.o $(obj)/build_error.o \ +$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/kernel/errno_generated.rs \ + $(obj)/alloc.o $(obj)/build_error.o \ $(obj)/libmacros.so $(obj)/bindings.o $(obj)/uapi.o FORCE +$(call if_changed_rule,rustc_library)
rust/kernel/error.rs +4 -54
64 lines not on this page · raw message
Signature · 1 line
2.46.0
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
In reply to Greg KHQuoted 9 lines · Greg KH
Quoted 7 lines · Yiyang Wu
Introduce Errno to Rust side code. Note that in current Rust For Linux, Errnos are implemented as
core::ffi::c_uintunit structs. However, EUCLEAN, a.k.a EFSCORRUPTED is missing from error crate.Since the
errno_basehasn't changed for over 13 years, This patch merely serves as a temporary workaround for the missing errno in the Rust For Linux.
Quoted 6 lines
Why not just add the missing errno to the core rust code instead? No need to define a whole new one for this.
thanks,
greg k-h
I have added all the missing errnos by autogenerating declare_err! in the preceding patches. Please check :)
Best Regards,
Yiyang Wu
Gary Guo
reviewerRe: [RFC PATCH 03/24] erofs: add Errno in Rust
In reply to Yiyang WuQuoted 9 lines · On Mon, 16 Sep 2024 21:56:13 +0800 Yiyang Wu <[email protected]> wrote:
Introduce Errno to Rust side code. Note that in current Rust For Linux, Errnos are implemented as
core::ffi::c_uintunit structs. However, EUCLEAN, a.k.a EFSCORRUPTED is missing from error crate.Since the
errno_basehasn't changed for over 13 years, This patch merely serves as a temporary workaround for the missing errno in the Rust For Linux.Signed-off-by Yiyang Wu <[email protected]>
As Greg said, please add missing errno that you need to kernel crate instead.
Also, it seems that you're building abstractions into EROFS directly without building a generic abstraction. We have been avoiding that. If there's an abstraction that you need and missing, please add that abstraction. In fact, there're a bunch of people trying to add FS support, please coordinate instead of rolling your own.
You also have been referencing kernel::bindings:: directly in various places in the patch series. The module is marked as #[doc(hidden)] for a reason -- it's not supposed to referenced directly. It's only exposed so that macros can reference them. In fact, we have a policy that direct reference to raw bindings are not allowed from drivers.
There're a few issues with this patch itself that I pointed out below, although as already said this would require big changes so most points are probably moot anyway.
Thanks,
Gary
Quoted 27 lines
--- fs/erofs/rust/erofs_sys.rs | 6 + fs/erofs/rust/erofs_sys/errnos.rs | 191 ++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 fs/erofs/rust/erofs_sys/errnos.rsfs/erofs/rust/erofs_sys.rs +6
-0@@ -19,4 +19,10 @@pub (crate )type Nid =u64 ;/// Erofs Super Offset to read the ondisk superblock pub (crate )const EROFS_SUPER_OFFSET :Off =1024 ; +/// PosixResult as a type alias to kernel::error::Result +/// to avoid naming conflicts. +pub (crate )type PosixResult <T > =Result <T ,Errno >; + +pub (crate )mod errnos;pub (crate )mod superblock; +pub (crate )use errnos::Errno ;fs/erofs/rust/erofs_sys/errnos.rs new file +0
-0
fs/erofs/rust/erofs_sys/errnos.rs line 9
Quoted 9 lines
fs/erofs/rust/erofs_sys/errnos.rs new file +9
-0@@ -0,0 +1,191 @@ +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later + +#[repr( i32 )] +#[non_exhaustive] +#[allow(clippy::upper_case_acronyms)] +#[derive( Debug , Copy , Clone , PartialEq )] +pub (crate )enum Errno { +NONE =0 ,
Why is NONE an error? No "error: operation completed successfully" please.
Quoted 31 lines
fs/erofs/rust/erofs_sys/errnos.rs new file +31
-0+ EPERM, + ENOENT, + ESRCH, + EINTR, + EIO, + ENXIO, + E2BIG, + ENOEXEC, + EBADF, + ECHILD, + EAGAIN, + ENOMEM, + EACCES, + EFAULT, + ENOTBLK, + EBUSY, + EEXIST, + EXDEV, + ENODEV, + ENOTDIR, + EISDIR, + EINVAL, + ENFILE, + EMFILE, + ENOTTY, + ETXTBSY, + EFBIG, + ENOSPC, + ESPIPE, + EROFS, + EMLINK,
fs/erofs/rust/erofs_sys/errnos.rs line 50
Quoted 10 lines
fs/erofs/rust/erofs_sys/errnos.rs new file +10
-0+ EPIPE, + EDOM, + ERANGE, + EDEADLK, + ENAMETOOLONG, + ENOLCK, + ENOSYS, + ENOTEMPTY, + ELOOP, + ENOMSG = 42,
This looks very fragile way to maintain an enum.
Quoted 91 lines
fs/erofs/rust/erofs_sys/errnos.rs new file +91
-0+ EIDRM, + ECHRNG, + EL2NSYNC, + EL3HLT, + EL3RST, + ELNRNG, + EUNATCH, + ENOCSI, + EL2HLT, + EBADE, + EBADR, + EXFULL, + ENOANO, + EBADRQC, + EBADSLT, + EBFONT = 59, + ENOSTR, + ENODATA, + ETIME, + ENOSR, + ENONET, + ENOPKG, + EREMOTE, + ENOLINK, + EADV, + ESRMNT, + ECOMM, + EPROTO, + EMULTIHOP, + EDOTDOT, + EBADMSG, + EOVERFLOW, + ENOTUNIQ, + EBADFD, + EREMCHG, + ELIBACC, + ELIBBAD, + ELIBSCN, + ELIBMAX, + ELIBEXEC, + EILSEQ, + ERESTART, + ESTRPIPE, + EUSERS, + ENOTSOCK, + EDESTADDRREQ, + EMSGSIZE, + EPROTOTYPE, + ENOPROTOOPT, + EPROTONOSUPPORT, + ESOCKTNOSUPPORT, + EOPNOTSUPP, + EPFNOSUPPORT, + EAFNOSUPPORT, + EADDRINUSE, + EADDRNOTAVAIL, + ENETDOWN, + ENETUNREACH, + ENETRESET, + ECONNABORTED, + ECONNRESET, + ENOBUFS, + EISCONN, + ENOTCONN, + ESHUTDOWN, + ETOOMANYREFS, + ETIMEDOUT, + ECONNREFUSED, + EHOSTDOWN, + EHOSTUNREACH, + EALREADY, + EINPROGRESS, + ESTALE, + EUCLEAN, + ENOTNAM, + ENAVAIL, + EISNAM, + EREMOTEIO, + EDQUOT, + ENOMEDIUM, + EMEDIUMTYPE, + ECANCELED, + ENOKEY, + EKEYEXPIRED, + EKEYREVOKED, + EKEYREJECTED, + EOWNERDEAD, + ENOTRECOVERABLE, + ERFKILL, + EHWPOISON, + EUNKNOWN,
fs/erofs/rust/erofs_sys/errnos.rs line 151
Quoted 10 lines
fs/erofs/rust/erofs_sys/errnos.rs new file +10
-0+} + +impl From <i32 >for Errno { +fn from (value:i32 ) ->Self { +if (-value) <=0 || (-value) >Errno ::EUNKNOWN as i32 { +Errno ::EUNKNOWN + }else { +// Safety: The value is guaranteed to be a valid errno and the memory +// layout is the same for both types. +unsafe { core::mem::transmute (value) }
This is just unsound. As evident from the fact that you need to manually specify a few constants, the errno enum doesn't cover all values from 1 to EUNKNOWN.
Quoted 40 lines
fs/erofs/rust/erofs_sys/errnos.rs new file +40
-0+ } + } +} + +impl From <Errno >for i32 { +fn from (value:Errno ) ->Self { + -(valueas i32 ) + } +} + +/// Replacement for ERR_PTR in Linux Kernel. +impl From <Errno >for *const core::ffi::c_void { +fn from (value:Errno ) ->Self { + (-(valueas core::ffi::c_long ))as *const core::ffi::c_void + } +} + +impl From <Errno >for *mut core::ffi::c_void { +fn from (value:Errno ) ->Self { + (-(valueas core::ffi::c_long ))as *mut core::ffi::c_void + } +} + +/// Replacement for PTR_ERR in Linux Kernel. +impl From <*const core::ffi::c_void >for Errno { +fn from (value: *const core::ffi::c_void ) ->Self { + (-(valueas i32 )).into () + } +} + +impl From <*mut core::ffi::c_void >for Errno { +fn from (value: *mut core::ffi::c_void ) ->Self { + (-(valueas i32 )).into () + } +} +/// Replacement for IS_ERR in Linux Kernel. +#[inline(always)] +pub (crate )fn is_value_err (value: *const core::ffi::c_void ) ->bool { + (valueas core::ffi::c_ulong ) >= (-4095 as core::ffi::c_long )as core::ffi::c_ulong +}
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Hi Gary,
Quoted 7 lines · Gary Guo
Quoted 4 lines · On Mon, 16 Sep 2024 21:56:13 +0800 Yiyang Wu <[email protected]> wrote:
Introduce Errno to Rust side code. Note that in current Rust For Linux, Errnos are implemented as
core::ffi::c_uintunit structs. However, EUCLEAN, a.k.a EFSCORRUPTED is missing from error crate.
Quoted 8 lines
Quoted 5 lines
Since the
errno_basehasn't changed for over 13 years, This patch merely serves as a temporary workaround for the missing errno in the Rust For Linux.Signed-off-by Yiyang Wu <[email protected]>As Greg said, please add missing errno that you need to kernel crate instead.
I've answered Greg about this in another email.
Quoted 5 lines
Also, it seems that you're building abstractions into EROFS directly without building a generic abstraction. We have been avoiding that. If there's an abstraction that you need and missing, please add that abstraction. In fact, there're a bunch of people trying to add FS
No, I'd like to try to replace some EROFS C logic first to Rust (by using EROFS C API interfaces) and try if Rust is really useful for a real in-tree filesystem. If Rust can improve EROFS security or performance (although I'm sceptical on performance), As an EROFS maintainer, I'm totally fine to accept EROFS Rust logic landed to help the whole filesystem better.
For Rust VFS abstraction, that is a different and indepenent story, Yiyang don't have any bandwidth on this due to his limited time. And I _also_ don't think an incomplete ROFS VFS Rust abstraction is useful to Linux community (because IMO for generic interface design, we need a global vision for all filesystems instead of just ROFSes. No existing user is not an excuse for an incomplete abstraction.)
If a reasonble Rust VFS abstraction landed, I think we will switch to use that, but as I said, they are completely two stories.
Quoted 7 lines
support, please coordinate instead of rolling your own.
You also have been referencing
kernel::bindings::directly in various places in the patch series. The module is marked as#[doc(hidden)]for a reason -- it's not supposed to referenced directly. It's only exposed so that macros can reference them. In fact, we have a policy that direct reference to raw bindings are not allowed from drivers.
This patch can be avoided if EUCLEAN is added to errno.
Thanks,
Gao Xiang
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Hi,
Thanks for the patch series. I think it's great that you want to use Rust for this filesystem.
Quoted 6 lines · Gao Xiang
Quoted 4 lines · Gary Guo
Also, it seems that you're building abstractions into EROFS directly without building a generic abstraction. We have been avoiding that. If there's an abstraction that you need and missing, please add that abstraction. In fact, there're a bunch of people trying to add FS
Quoted 6 lines
No, I'd like to try to replace some EROFS C logic first to Rust (by using EROFS C API interfaces) and try if Rust is really useful for a real in-tree filesystem. If Rust can improve EROFS security or performance (although I'm sceptical on performance), As an EROFS maintainer, I'm totally fine to accept EROFS Rust logic landed to help the whole filesystem better.
As Gary already said, we have been using a different approach and it has served us well. Your approach of calling directly into C from the driver can be used to create a proof of concept, but in our opinion it is not something that should be put into mainline. That is because calling C from Rust is rather complicated due to the many nuanced features that Rust provides (for example the safety requirements of references). Therefore moving the dangerous parts into a central location is crucial for making use of all of Rust's advantages inside of your code.
Quoted 2 lines
For Rust VFS abstraction, that is a different and indepenent story, Yiyang don't have any bandwidth on this due to his limited time.
This seems a bit weird, you have the bandwidth to write your own abstractions, but not use the stuff that has already been developed?
I have quickly glanced over the patchset and the abstractions seem rather immature, not general enough for other filesystems to also take advantage of them. They also miss safety documentation and are in general poorly documented.
Additionally, all of the code that I saw is put into the fs/erofs and rust/erofs_sys directories. That way people can't directly benefit from your code, put your general abstractions into the kernel crate. Soon we will be split the kernel crate, I could imagine that we end up with an fs crate, when that happens, we would put those abstractions there.
As I don't have the bandwidth to review two different sets of filesystem abstractions, I can only provide you with feedback if you use the existing abstractions.
Quoted 2 lines
And I _also_ don't think an incomplete ROFS VFS Rust abstraction is useful to Linux community
IIRC Wedson created ROFS VFS abstractions before going for the full filesystem. So it would definitely be useful for other read-only filesystems (as well as filesystems that also allow writing, since last time I checked, they often also support reading).
Quoted 4 lines
(because IMO for generic interface
design, we need a global vision for all filesystems instead of just ROFSes. No existing user is not an excuse for an incomplete abstraction.)
Yes we need a global vision, but if you would use the existing abstractions, then you would participate in this global vision.
Sorry for repeating this point so many times, but it is *really* important that we don't have multiple abstractions for the same thing.
Quoted 2 lines
If a reasonble Rust VFS abstraction landed, I think we will switch to use that, but as I said, they are completely two stories.
For them to land, there has to be some kind of user. For example, a rust reference driver, or a new filesystem. For example this one.
---
Cheers,
Benno
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Hi Benno,
Quoted 19 lines · Benno Lossin
Hi,
Thanks for the patch series. I think it's great that you want to use Rust for this filesystem.
Quoted 6 lines · Gao Xiang
Quoted 4 lines · Gary Guo
Also, it seems that you're building abstractions into EROFS directly without building a generic abstraction. We have been avoiding that. If there's an abstraction that you need and missing, please add that abstraction. In fact, there're a bunch of people trying to add FS
Quoted 6 lines
No, I'd like to try to replace some EROFS C logic first to Rust (by using EROFS C API interfaces) and try if Rust is really useful for a real in-tree filesystem. If Rust can improve EROFS security or performance (although I'm sceptical on performance), As an EROFS maintainer, I'm totally fine to accept EROFS Rust logic landed to help the whole filesystem better.
Quoted 8 lines
As Gary already said, we have been using a different approach and it has served us well. Your approach of calling directly into C from the driver can be used to create a proof of concept, but in our opinion it is not something that should be put into mainline. That is because calling C from Rust is rather complicated due to the many nuanced features that Rust provides (for example the safety requirements of references). Therefore moving the dangerous parts into a central location is crucial for making use of all of Rust's advantages inside of your code.
I'm not quite sure about your point honestly. In my opinion, there is nothing different to use Rust _within a filesystem_ or _within a driver_ or _within a Linux subsystem_ as long as all negotiated APIs are audited.
Otherwise, it means Rust will never be used to write Linux core parts such as MM, VFS or block layer. Does this point make sense? At least, Rust needs to get along with the existing C code (in an audited way) rather than refuse C code.
My personal idea about Rust: I think Rust is just another _language tool_ for the Linux kernel which could save us time and make the kernel development better.
Or I wonder why not writing a complete new Rust stuff instead rather than living in the C world?
Quoted 6 lines
Quoted 3 lines
For Rust VFS abstraction, that is a different and indepenent story, Yiyang don't have any bandwidth on this due to his limited time.
This seems a bit weird, you have the bandwidth to write your own abstractions, but not use the stuff that has already been developed?
It's not written by me, Yiyang is still an undergraduate tudent. It's his research project and I don't think it's his responsibility to make an upstreamable VFS abstraction.
Quoted 3 lines
I have quickly glanced over the patchset and the abstractions seem rather immature, not general enough for other filesystems to also take
I don't have enough time to take a full look of this patchset too due to other ongoing work for now (Rust EROFS is not quite a high priority stuff for me).
And that's why it's called "RFC PATCH".
Quoted 1 line
advantage of them. They also miss safety documentation and are in
I don't think it needs to be general enough, since we'd like to use the new Rust language tool within a subsystem.
So why it needs to take care of other filesystems? Again, I'm not working on a full VFS abstriction.
Yes, this patchset is not perfect. But I've asked Yiyang to isolate all VFS structures as much as possible, but it seems that it still touches something.
Quoted 1 line
general poorly documented.
Okay, I think it can be improved then if you give more detailed hints.
Quoted 11 lines
Additionally, all of the code that I saw is put into the
fs/erofsandrust/erofs_sysdirectories. That way people can't directly benefit from your code, put your general abstractions into the kernel crate. Soon we will be split the kernel crate, I could imagine that we end up with anfscrate, when that happens, we would put those abstractions there.As I don't have the bandwidth to review two different sets of filesystem abstractions, I can only provide you with feedback if you use the existing abstractions.
I think Rust is just a tool, if you could have extra time to review our work, that would be wonderful! Many thanks then.
However, if you don't have time to review, IMHO, Rust is just a tool, I think each subsystem can choose to use Rust in their codebase, or I'm not sure what's your real point is?
Quoted 8 lines
Quoted 3 lines
And I _also_ don't think an incomplete ROFS VFS Rust abstraction is useful to Linux community
IIRC Wedson created ROFS VFS abstractions before going for the full filesystem. So it would definitely be useful for other read-only filesystems (as well as filesystems that also allow writing, since last time I checked, they often also support reading).
Leaving aside everything else, an incomplete Rust read-only VFS abstraction itself is just an unsafe stuff.
Quoted 11 lines
Quoted 5 lines
(because IMO for generic interface
design, we need a global vision for all filesystems instead of just ROFSes. No existing user is not an excuse for an incomplete abstraction.)Yes we need a global vision, but if you would use the existing abstractions, then you would participate in this global vision.
Sorry for repeating this point so many times, but it is *really* important that we don't have multiple abstractions for the same thing.
I've expressed my viewpoint.
Quoted 6 lines
Quoted 3 lines
If a reasonble Rust VFS abstraction landed, I think we will switch to use that, but as I said, they are completely two stories.
For them to land, there has to be some kind of user. For example, a rust reference driver, or a new filesystem. For example this one.
Without a full proper VFS abstraction, it's just broken and needs to be refactored. And that will be painful to all users then.
=======
In the end,
Other thoughts, comments are helpful here since I wonder how "Rust -for-Linux" works in the long term, and decide whether I will work on Kernel Rust or not at least in the short term.
Thanks,
Gao Xiang
Quoted 5 lines
---
Cheers,
Benno
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Quoted 31 lines · Gao Xiang
Hi Benno,
Quoted 19 lines · Benno Lossin
Hi,
Thanks for the patch series. I think it's great that you want to use Rust for this filesystem.
Quoted 6 lines · Gao Xiang
Quoted 4 lines · Gary Guo
Also, it seems that you're building abstractions into EROFS directly without building a generic abstraction. We have been avoiding that. If there's an abstraction that you need and missing, please add that abstraction. In fact, there're a bunch of people trying to add FS
Quoted 6 lines
No, I'd like to try to replace some EROFS C logic first to Rust (by using EROFS C API interfaces) and try if Rust is really useful for a real in-tree filesystem. If Rust can improve EROFS security or performance (although I'm sceptical on performance), As an EROFS maintainer, I'm totally fine to accept EROFS Rust logic landed to help the whole filesystem better.
Quoted 8 lines
As Gary already said, we have been using a different approach and it has served us well. Your approach of calling directly into C from the driver can be used to create a proof of concept, but in our opinion it is not something that should be put into mainline. That is because calling C from Rust is rather complicated due to the many nuanced features that Rust provides (for example the safety requirements of references). Therefore moving the dangerous parts into a central location is crucial for making use of all of Rust's advantages inside of your code.
Quoted 4 lines
I'm not quite sure about your point honestly. In my opinion, there is nothing different to use Rust
_withinafilesystem_or_withinadriver_or_withina Linuxsubsystem_as long as all negotiated APIs are audited.
To us there is a big difference: If a lot of functions in an API are unsafe without being inherent from the problem that it solves, then it's a bad API.
Quoted 4 lines
Otherwise, it means Rust will never be used to write Linux core parts such as MM, VFS or block layer. Does this point make sense? At least, Rust needs to get along with the existing C code (in an audited way) rather than refuse C code.
I am neither requiring you to write solely safe code, nor am I banning interacting with the C side. What we mean when we talk about abstractions is that we want to minimize the Rust code that directly interfaces with C. Rust-to-Rust interfaces can be a lot safer and are easier to implement correctly.
Quoted 3 lines
My personal idea about Rust: I think Rust is just another
_languagetool_for the Linux kernel which could save us time and make the kernel development better.
Yes, but we do have conventions, rules and guidelines for writing such code. C code also has them. If you want/need to break them, there should be a good reason to do so. I don't see one in this instance.
Quoted 2 lines
Or I wonder why not writing a complete new Rust stuff instead rather than living in the C world?
There are projects that do that yes. But Rust-for-Linux is about bringing Rust to the kernel and part of that is coming up with good conventions and rules.
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
For Rust VFS abstraction, that is a different and indepenent story, Yiyang don't have any bandwidth on this due to his limited time.
This seems a bit weird, you have the bandwidth to write your own abstractions, but not use the stuff that has already been developed?
It's not written by me, Yiyang is still an undergraduate tudent. It's his research project and I don't think it's his responsibility to make an upstreamable VFS abstraction.
That is fair, but he wouldn't have to start from scratch, Wedsons abstractions were good enough for him to write a Rust version of ext2. In addition, tarfs and puzzlefs also use those bindings.
To me it sounds as if you have not taken the time to try to make it work with the existing abstractions. Have you tried reaching out to Ariel? He is working on puzzlefs and might have some insight to give you. Sadly Wedson has left the project, so someone will have to pick up his work.
I hope that you understand that we can't have two abstractions for the same C API. It confuses people which to use, some features might only be available in one version and others only in the other. It would be a total mess. It's just like the rule for no duplicated drivers that you have on the C side.
People (mostly Wedson) also have put in a lot of work into making the VFS abstractions good. Why ignore all of that?
Quoted 8 lines
Quoted 2 lines
I have quickly glanced over the patchset and the abstractions seem rather immature, not general enough for other filesystems to also take
I don't have enough time to take a full look of this patchset too due to other ongoing work for now (Rust EROFS is not quite a high priority stuff for me).
And that's why it's called "RFC PATCH".
Yeah I saw the RFC title. I just wanted to communicate early that I would not review it if it were a normal patch. In fact, I would advise against taking the patch, due to the reasons I outlined.
Quoted 7 lines
Quoted 1 line
advantage of them. They also miss safety documentation and are in
I don't think it needs to be general enough, since we'd like to use the new Rust language tool within a subsystem.
So why it needs to take care of other filesystems? Again, I'm not working on a full VFS abstriction.
And that's OK, feel free to just pick the parts of the existing VFS that you need and extend as you (or your student) see fit. What you said yourself is that we need a global vision for VFS abstractions. If you only use a subset of them, then you only care about that subset, other people can extend it if they need. If everyone would roll their own abstractions without communicating, then how would we create a global vision?
Quoted 3 lines
Yes, this patchset is not perfect. But I've asked Yiyang to isolate all VFS structures as much as possible, but it seems that it still touches something.
It would already be a big improvement to put the VFS structures into the kernel crate. Because then everyone can benefit from your work.
Quoted 16 lines
Quoted 1 line
general poorly documented.
Okay, I think it can be improved then if you give more detailed hints.
Quoted 11 lines
Additionally, all of the code that I saw is put into the
fs/erofsandrust/erofs_sysdirectories. That way people can't directly benefit from your code, put your general abstractions into the kernel crate. Soon we will be split the kernel crate, I could imagine that we end up with anfscrate, when that happens, we would put those abstractions there.As I don't have the bandwidth to review two different sets of filesystem abstractions, I can only provide you with feedback if you use the existing abstractions.
Quoted 6 lines
I think Rust is just a tool, if you could have extra time to review our work, that would be wonderful! Many thanks then.
However, if you don't have time to review, IMHO, Rust is just a tool, I think each subsystem can choose to use Rust in their codebase, or I'm not sure what's your real point is?
I don't want to prevent or discourage you from using Rust in the kernel. In fact, I can't prevent you from putting this in, since after all you are the maintainer.
What I can do, is advise against not using abstractions. That has been our philosophy since very early on. They are the reason that you can write PHY drivers without any unsafe code whatsoever *today*. I think that is an impressive feat and our recipe for success.
We even have this in our documentation:
https://docs.kernel.org/rust/general-information.html#abstractions-vs-bindings
My real point is that I want Rust to succeed in the kernel. I strongly believe that good abstractions (in the sense that you can do as much as possible using only safe Rust) are a crucial factor.
I and others from the RfL team can help you if you (or your student) have any Rust related questions for the abstractions. Feel free to reach out.
Maybe Miguel can say more on this matter, since he was at the maintainers summit, but our takeaways essentially are that we want maintainers to experiment with Rust. And if you don't have any real users, then breaking the Rust code is fine.
Though I think that with breaking we mean that changes to the C side prevent the Rust side from working, not shipping Rust code without abstractions.
We might be able to make an exception to the "your driver can only use abstractions" rule, but only with the promise that the subsystem is working towards creating suitable abstractions and replacing the direct C accesses with that.
I personally think that we should not make that the norm, instead try to create the minimal abstraction and minimal driver (without directly calling C) that you need to start. Of course this might not work, the "minimal driver" might need to be rather complex for you to start, but I don't know your subsystem to make that judgement.
Quoted 10 lines
Quoted 7 lines
Quoted 2 lines
And I _also_ don't think an incomplete ROFS VFS Rust abstraction is useful to Linux community
IIRC Wedson created ROFS VFS abstractions before going for the full filesystem. So it would definitely be useful for other read-only filesystems (as well as filesystems that also allow writing, since last time I checked, they often also support reading).
Leaving aside everything else, an incomplete Rust read-only VFS abstraction itself is just an unsafe stuff.
I don't understand what you want to say.
Quoted 17 lines
Quoted 10 lines
Quoted 4 lines
(because IMO for generic interface
design, we need a global vision for all filesystems instead of just ROFSes. No existing user is not an excuse for an incomplete abstraction.)Yes we need a global vision, but if you would use the existing abstractions, then you would participate in this global vision.
Sorry for repeating this point so many times, but it is *really* important that we don't have multiple abstractions for the same thing.
I've expressed my viewpoint.
Quoted 4 lines
Quoted 3 lines
If a reasonble Rust VFS abstraction landed, I think we will switch to use that, but as I said, they are completely two stories.
Quoted 6 lines
Quoted 2 lines
For them to land, there has to be some kind of user. For example, a rust reference driver, or a new filesystem. For example this one.
Without a full proper VFS abstraction, it's just broken and needs to be refactored. And that will be painful to all users then.
I also don't understand your point here. What is broken, this EROFS implementation? Why will it be painful to refactor?
Quoted 6 lines
=======
In the end,Other thoughts, comments are helpful here since I wonder how "Rust -for-Linux" works in the long term, and decide whether I will work on Kernel Rust or not at least in the short term.
The longterm goal is to make everything that is possible in C, possible in Rust. For more info, please take a look at the kernel summit talk by Miguel Ojeda.
However, we can only reach that longterm goal if maintainers are willing and ready to put Rust into their subsystems (either by knowing/learning Rust themselves or by having a co-maintainer that does just the Rust part). So you wanting to experiment is great. I appreciate that you also have a student working on this. Still, I think we should follow our guidelines and create abstractions in order to require as little unsafe code as possible.
---
Cheers,
Benno
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Quoted 32 lines · Benno Lossin
Quoted 31 lines · Gao Xiang
Hi Benno,
Quoted 27 lines · Benno Lossin
Hi,
Thanks for the patch series. I think it's great that you want to use Rust for this filesystem.
Quoted 6 lines · Gao Xiang
Quoted 4 lines · Gary Guo
Also, it seems that you're building abstractions into EROFS directly without building a generic abstraction. We have been avoiding that. If there's an abstraction that you need and missing, please add that abstraction. In fact, there're a bunch of people trying to add FS
Quoted 6 lines
No, I'd like to try to replace some EROFS C logic first to Rust (by using EROFS C API interfaces) and try if Rust is really useful for a real in-tree filesystem. If Rust can improve EROFS security or performance (although I'm sceptical on performance), As an EROFS maintainer, I'm totally fine to accept EROFS Rust logic landed to help the whole filesystem better.
As Gary already said, we have been using a different approach and it has served us well. Your approach of calling directly into C from the driver can be used to create a proof of concept, but in our opinion it is not something that should be put into mainline. That is because calling C from Rust is rather complicated due to the many nuanced features that Rust provides (for example the safety requirements of references). Therefore moving the dangerous parts into a central location is crucial for making use of all of Rust's advantages inside of your code.
Quoted 8 lines
Quoted 4 lines
I'm not quite sure about your point honestly. In my opinion, there is nothing different to use Rust
_withinafilesystem_or_withinadriver_or_withina Linuxsubsystem_as long as all negotiated APIs are audited.To us there is a big difference: If a lot of functions in an API are
unsafewithout being inherent from the problem that it solves, then it's a bad API.
Which one? If you point it out, we will update the EROFS kernel APIs then.
Quoted 10 lines
Quoted 5 lines
Otherwise, it means Rust will never be used to write Linux core parts such as MM, VFS or block layer. Does this point make sense? At least, Rust needs to get along with the existing C code (in an audited way) rather than refuse C code.
I am neither requiring you to write solely safe code, nor am I banning interacting with the C side. What we mean when we talk about abstractions is that we want to minimize the Rust code that directly interfaces with C. Rust-to-Rust interfaces can be a lot safer and are
We will definitly minimize the API interface between Rust and C in EROFS.
And it can be done incrementally, why not? I assume your world is not pure C and pure Rust as for the Rust for Linux project, no?
Quoted 9 lines
easier to implement correctly.
Quoted 3 lines
My personal idea about Rust: I think Rust is just another
_languagetool_for the Linux kernel which could save us time and make the kernel development better.Yes, but we do have conventions, rules and guidelines for writing such code. C code also has them. If you want/need to break them, there should be a good reason to do so. I don't see one in this instance.
Quoted 6 lines
Quoted 2 lines
Quoted 1 line
Or I wonder why not writing a complete new Rust stuff instead rather
than living in the C world?
There are projects that do that yes. But Rust-for-Linux is about bringing Rust to the kernel and part of that is coming up with good conventions and rules.
Which rule is broken? Was they discussed widely around the Linux world?
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
Quoted 3 lines
For Rust VFS abstraction, that is a different and indepenent story, Yiyang don't have any bandwidth on this due to his limited time.
This seems a bit weird, you have the bandwidth to write your own abstractions, but not use the stuff that has already been developed?
Quoted 6 lines
Quoted 3 lines
It's not written by me, Yiyang is still an undergraduate tudent. It's his research project and I don't think it's his responsibility to make an upstreamable VFS abstraction.
That is fair, but he wouldn't have to start from scratch, Wedsons abstractions were good enough for him to write a Rust version of ext2.
The Wedson one is just broken, I assume that you've read
https://lwn.net/Articles/978738/ ?
The initial Linux VFS C version is already for generic read-write use.
Quoted 1 line
In addition, tarfs and puzzlefs also use those bindings.
These are both toy fses, I don't know who will use these two fses for their customers.
Quoted 3 lines
To me it sounds as if you have not taken the time to try to make it work with the existing abstractions. Have you tried reaching out to Ariel? He is working on puzzlefs and might have some insight to give you. Sadly
IMHO, puzzlefs is another Rust incomplete clone of EROFS, I could tell him what EROFS currently do.
I'm very happy to collaborate with him to work on his use cases (and tell him why EROFS can already be used for his use cases), just
like the previous ComposeFS discussion.
There are enough FS projects which reinvents in-tree fses without enough good reasons (for example, performance or design): ZUFS, FamFS, ComposeFS.
Tarfs (here tar is not the real tar format), and Puzzlefs are two special one just because they are written in Rust. But other than that they are just incomplete approach to EROFS.
I do hope Ariel could attend LSF/MM/BPF to discuss his use cases with filesystem developpers. And I'm very happy to collaborate with him .
Quoted 1 line
Wedson has left the project, so someone will have to pick up his work.
It's not necessary to be Yiyang, since he's interested in EROFS only.
Quoted 9 lines
I hope that you understand that we can't have two abstractions for the same C API. It confuses people which to use, some features might only be available in one version and others only in the other. It would be a total mess. It's just like the rule for no duplicated drivers that you have on the C side.
People (mostly Wedson) also have put in a lot of work into making the VFS abstractions good. Why ignore all of that?
How good? TBH, I think there could be something left eventually, but the current prososed Rust VFS abstraction is just broken.
I don't think the current abstraction is of any use to be upstreamed, at least, it should be driven with a generic read-write filesystem, and resolve lifetime issues during development (for example, just like Al mentioned d_name and d_parent, etc.)
Because the initial Linux VFS C version is completely out of minix fs, rather than an incomplete broken one just for some toy (I don't know how to express more accurately, since each upstream filesystem should have strong use cases and users, but tarfs and puzzlefs are both not.)
Otherwise, all the broken Rust VFS users will be painful for bugs and endless API refactering.
Quoted 8 lines
Quoted 8 lines
Quoted 3 lines
I have quickly glanced over the patchset and the abstractions seem rather immature, not general enough for other filesystems to also take
I don't have enough time to take a full look of this patchset too due to other ongoing work for now (Rust EROFS is not quite a high priority stuff for me).
Quoted 5 lines
Quoted 1 line
And that's why it's called "RFC PATCH".
Yeah I saw the RFC title. I just wanted to communicate early that I would not review it if it were a normal patch. In fact, I would advise against taking the patch, due to the reasons I outlined.
You reason currently is still not valid. IMO, again, Rust is just a tool, you cannot forbid a real Linux subsystem to use Rust as an experiment.
Or what's your real point? An Rust gatekeeper?
Quoted 9 lines
Quoted 8 lines
Quoted 2 lines
advantage of them. They also miss safety documentation and are in
I don't think it needs to be general enough, since we'd like to use the new Rust language tool within a subsystem.
So why it needs to take care of other filesystems? Again, I'm not working on a full VFS abstriction.
Quoted 7 lines
And that's OK, feel free to just pick the parts of the existing VFS that you need and extend as you (or your student) see fit. What you said yourself is that we need a global vision for VFS abstractions. If you only use a subset of them, then you only care about that subset, other people can extend it if they need. If everyone would roll their own abstractions without communicating, then how would we create a global vision?
No. We don't roll our own abstraction, instead we define a clear C <-> Rust boundary of EROFS APIs, just like "fs/xfs/libxfs" if you could take a look.
Quoted 7 lines
Quoted 4 lines
Yes, this patchset is not perfect. But I've asked Yiyang to isolate all VFS structures as much as possible, but it seems that it still touches something.
It would already be a big improvement to put the VFS structures into the kernel crate. Because then everyone can benefit from your work.
Again, that is not Yiyang's interest. Which is just like to sell something you don't want, I don't think it's reasonable.
Quoted 20 lines
Quoted 20 lines
Quoted 2 lines
general poorly documented.
Okay, I think it can be improved then if you give more detailed hints.
Quoted 11 lines
Additionally, all of the code that I saw is put into the
fs/erofsandrust/erofs_sysdirectories. That way people can't directly benefit from your code, put your general abstractions into the kernel crate. Soon we will be split the kernel crate, I could imagine that we end up with anfscrate, when that happens, we would put those abstractions there.As I don't have the bandwidth to review two different sets of filesystem abstractions, I can only provide you with feedback if you use the existing abstractions.
I think Rust is just a tool, if you could have extra time to review our work, that would be wonderful! Many thanks then.
Quoted 7 lines
Quoted 3 lines
However, if you don't have time to review, IMHO, Rust is just a tool, I think each subsystem can choose to use Rust in their codebase, or I'm not sure what's your real point is?
I don't want to prevent or discourage you from using Rust in the kernel. In fact, I can't prevent you from putting this in, since after all you are the maintainer.
I do think you're discouraging anyone to use Rust in their codebase, because I've said we _will_ form a good abstraction in our codebase.
But you're just selling another stuff forcely.
Quoted 3 lines
What I can do, is advise against not using abstractions. That has been our philosophy since very early on. They are the reason that you can write PHY drivers without any
unsafecode whatsoever *today*. I think
I don't think filesystems are comparable to some PHY drivers. If you take ext4, it's more than 65000 line, and XFS, that is almost 78000.
I think filesystems can have a way to be reimplmented in Rust incrementally, rather than purely black and write world.
Quoted 8 lines
that is an impressive feat and our recipe for success.
We even have this in our documentation:
https://docs.kernel.org/rust/general-information.html#abstractions-vs-bindingsMy real point is that I want Rust to succeed in the kernel. I strongly believe that good abstractions (in the sense that you can do as much as possible using only safe Rust) are a crucial factor.
On my side, you are just isolating any useful subsystem to try to use Rust and
“Leaf” modules (e.g. drivers) should not use the C bindings directly
is unreasonable for filesystems because it cannot be done in one shot.
In addition, there are a lot ongoing features on both C and Rust side, you need at least a fallback to make end users happy with a unique feature view rather than just return "it's broken".
Users don't care C or Rust (they only care full functionality) but only developers care. And mixing up two different things (VFS abstraction and use Rust in the codebase) is not good to RFL success.
IMHO, this is "Rust for Linux" not "Linux for Rust".
Quoted 5 lines
I and others from the RfL team can help you if you (or your student) have any Rust related questions for the abstractions. Feel free to reach out.
Quoted 7 lines
Maybe Miguel can say more on this matter, since he was at the maintainers summit, but our takeaways essentially are that we want maintainers to experiment with Rust. And if you don't have any real users, then breaking the Rust code is fine.
Though I think that with breaking we mean that changes to the C side prevent the Rust side from working, not shipping Rust code without abstractions.
I think you're still mixing them up.
Quoted 5 lines
We might be able to make an exception to the "your driver can only use abstractions" rule, but only with the promise that the subsystem is working towards creating suitable abstractions and replacing the direct C accesses with that.
I don't think those rules are reasonable for RFL success, honestly.
You are artificially isolating the Linux C and Rust world, not from Linux users or Linux ecosystem perspective, but only from some developer perspersive.
Good luck, anyway.
Quoted 6 lines
I personally think that we should not make that the norm, instead try to create the minimal abstraction and minimal driver (without directly calling C) that you need to start. Of course this might not work, the "minimal driver" might need to be rather complex for you to start, but I don't know your subsystem to make that judgement.
...
Quoted 7 lines
Quoted 4 lines
Without a full proper VFS abstraction, it's just broken and needs to be refactored. And that will be painful to all users then.
I also don't understand your point here. What is broken, this EROFS implementation? Why will it be painful to refactor?
I've said earlier.
Quoted 10 lines
Quoted 7 lines
=======
In the end,Other thoughts, comments are helpful here since I wonder how "Rust -for-Linux" works in the long term, and decide whether I will work on Kernel Rust or not at least in the short term.
The longterm goal is to make everything that is possible in C, possible in Rust. For more info, please take a look at the kernel summit talk by
But you're disallowing Rust in the codebase.
Quoted 8 lines
Miguel Ojeda.
However, we can only reach that longterm goal if maintainers are willing and ready to put Rust into their subsystems (either by knowing/learning Rust themselves or by having a co-maintainer that does just the Rust part). So you wanting to experiment is great. I appreciate that you also have a student working on this. Still, I think we should follow our guidelines and create abstractions in order to require as littleunsafecode as possible.
I've expressed my point. I don't think some guideline could bring success to RFL. Since many subsystems needs an incremental way, not just a black-or-white thing.
Thanks,
Gao Xiang
Quoted 4 lines
---
Cheers,
Benno
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Quoted 94 lines · Gao Xiang
Quoted 32 lines · Benno Lossin
Quoted 31 lines · Gao Xiang
Hi Benno,
Quoted 27 lines · Benno Lossin
Hi,
Thanks for the patch series. I think it's great that you want to use Rust for this filesystem.
Quoted 6 lines · Gao Xiang
Quoted 4 lines · Gary Guo
Also, it seems that you're building abstractions into EROFS directly without building a generic abstraction. We have been avoiding that. If there's an abstraction that you need and missing, please add that abstraction. In fact, there're a bunch of people trying to add FS
Quoted 6 lines
No, I'd like to try to replace some EROFS C logic first to Rust (by using EROFS C API interfaces) and try if Rust is really useful for a real in-tree filesystem. If Rust can improve EROFS security or performance (although I'm sceptical on performance), As an EROFS maintainer, I'm totally fine to accept EROFS Rust logic landed to help the whole filesystem better.
As Gary already said, we have been using a different approach and it has served us well. Your approach of calling directly into C from the driver can be used to create a proof of concept, but in our opinion it is not something that should be put into mainline. That is because calling C from Rust is rather complicated due to the many nuanced features that Rust provides (for example the safety requirements of references). Therefore moving the dangerous parts into a central location is crucial for making use of all of Rust's advantages inside of your code.
Quoted 8 lines
Quoted 4 lines
I'm not quite sure about your point honestly. In my opinion, there is nothing different to use Rust
_withinafilesystem_or_withinadriver_or_withina Linuxsubsystem_as long as all negotiated APIs are audited.To us there is a big difference: If a lot of functions in an API are
unsafewithout being inherent from the problem that it solves, then it's a bad API.Which one? If you point it out, we will update the EROFS kernel APIs then.
Quoted 10 lines
Quoted 5 lines
Otherwise, it means Rust will never be used to write Linux core parts such as MM, VFS or block layer. Does this point make sense? At least, Rust needs to get along with the existing C code (in an audited way) rather than refuse C code.
I am neither requiring you to write solely safe code, nor am I banning interacting with the C side. What we mean when we talk about abstractions is that we want to minimize the Rust code that directly interfaces with C. Rust-to-Rust interfaces can be a lot safer and are
We will definitly minimize the API interface between Rust and C in EROFS.
And it can be done incrementally, why not? I assume your world is not pure C and pure Rust as for the Rust for Linux project, no?
Quoted 9 lines
easier to implement correctly.
Quoted 3 lines
My personal idea about Rust: I think Rust is just another
_languagetool_for the Linux kernel which could save us time and make the kernel development better.Yes, but we do have conventions, rules and guidelines for writing such code. C code also has them. If you want/need to break them, there should be a good reason to do so. I don't see one in this instance.
Quoted 6 lines
Quoted 2 lines
Quoted 1 line
Or I wonder why not writing a complete new Rust stuff instead rather
than living in the C world?
There are projects that do that yes. But Rust-for-Linux is about bringing Rust to the kernel and part of that is coming up with good conventions and rules.
Which rule is broken? Was they discussed widely around the Linux world?
Quoted 11 lines
Quoted 10 lines
Quoted 6 lines
Quoted 3 lines
For Rust VFS abstraction, that is a different and indepenent story, Yiyang don't have any bandwidth on this due to his limited time.
This seems a bit weird, you have the bandwidth to write your own abstractions, but not use the stuff that has already been developed?
It's not written by me, Yiyang is still an undergraduate tudent. It's his research project and I don't think it's his responsibility to make an upstreamable VFS abstraction.
Quoted 5 lines
Quoted 2 lines
That is fair, but he wouldn't have to start from scratch, Wedsons abstractions were good enough for him to write a Rust version of ext2.
The Wedson one is just broken, I assume that you've read
https://lwn.net/Articles/978738/ ?
Yes, and if you see the patches on linux-fsdevel, people are working to get these vfs bindings correct for any filesystem to use. Please review them and see if they will work for you for erofs, as "burying" the binding in just one filesystem is not a good idea.
Quoted 4 lines
Quoted 1 line
In addition, tarfs and puzzlefs also use those bindings.
These are both toy fses, I don't know who will use these two fses for their customers.
tarfs is being used by real users as it solves a need they have today. And it's a good example of how the vfs bindings would work, although in a very simple way. You have to start somewhere :)
Quoted 9 lines
Quoted 8 lines
Miguel Ojeda.
However, we can only reach that longterm goal if maintainers are willing and ready to put Rust into their subsystems (either by knowing/learning Rust themselves or by having a co-maintainer that does just the Rust part). So you wanting to experiment is great. I appreciate that you also have a student working on this. Still, I think we should follow our guidelines and create abstractions in order to require as littleunsafecode as possible.
Quoted 3 lines
I've expressed my point. I don't think some
guidelinecould bring success to RFL. Since many subsystems needs an incremental way, not just a black-or-white thing.
Incremental is good, and if you want to use a .rs file or two in the middle of your module, that's fine. But please don't try to implement bindings to common kernel data structures like inodes and dentries and superblocks if at all possible and ignore the work that others are doing in this area as that's just duplicated work and will cause more confusion over time.
It's the same for drivers, I will object strongly if someone attempted to create a USB binding for 'struct usb_interface' in the middle of a USB driver and instead insist they work on a generic binding that can be used by all USB drivers. I imagine the VFS maintainers have the same opinion on their apis as well for valid reasons.
thanks,
greg k-h
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Hi Greg,
Quoted 3 lines · Greg Kroah-Hartman
Quoted 2 lines · Gao Xiang
...
Quoted 15 lines
Quoted 15 lines
Quoted 14 lines
Quoted 11 lines
Quoted 7 lines
Quoted 4 lines
For Rust VFS abstraction, that is a different and indepenent story, Yiyang don't have any bandwidth on this due to his limited time.
This seems a bit weird, you have the bandwidth to write your own abstractions, but not use the stuff that has already been developed?
It's not written by me, Yiyang is still an undergraduate tudent. It's his research project and I don't think it's his responsibility to make an upstreamable VFS abstraction.
That is fair, but he wouldn't have to start from scratch, Wedsons abstractions were good enough for him to write a Rust version of ext2.
Quoted 7 lines
Quoted 2 lines
The Wedson one is just broken, I assume that you've read
https://lwn.net/Articles/978738/ ?Yes, and if you see the patches on linux-fsdevel, people are working to get these vfs bindings correct for any filesystem to use. Please review them and see if they will work for you for erofs, as "burying" the binding in just one filesystem is not a good idea.
Thanks for the reply!
I do think the first Rust filesystem should be ext2 or other simple read-write fses due to many VFS member lifetime concerns as other filesystem developpers suggested before [1], otherwise honestly the VFS abstraction will be refactoredagain and again just due to limited vision and broken functionality, I do think which way is not how currently new C Linux kernel APIs are proposed too (e.g. carefully review all potential use cases).
[1] https://lore.kernel.org/linux-fsdevel/ZZ3GeehAw%[email protected]/
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
In addition, tarfs and puzzlefs also use those bindings.
These are both toy fses, I don't know who will use these two fses for their customers.
tarfs is being used by real users as it solves a need they have today. And it's a good example of how the vfs bindings would work, although in a very simple way. You have to start somewhere :)
EROFS has resolved the same functionality upstream in 2023, see [2]
``` Replacing tar or cpio archives with a filesystem is a potential use case for EROFS. There has been a proposal from the confidential-computing community for a kernel tarfs filesystem, which would allow guest VMs to efficiently mount a tar file directly. But EROFS would be a better choice, he said. There is a proof-of-concept patch set that allows directly mounting a downloaded tar file using EROFS that performs better than unpacking the tarball to ext4, then mounting it in the guest using overlayfs. ```
Honestly, I've kept doing very friendly public/private communitation with Wedson in the confidential-computing community to see if there could be a collaboration for our tar direct mount use cases, but he just ignored my suggestion [3] and keep on his "tarfs" development (even this "tarfs" has no relationship with the standard POSIX tar/pax format because you cannot mount a real POSIX tar/pax by his implementation.)
So my concern is just as below:
1) EROFS can already work well for his "tarfs" use cases, so there is already an in-tree stuff works without any issue;
2) No matter from his "tarfs" current on-disk format, and on-disk extendability perspersive, I think it will be used for a very narrow use case. So in the long term, it could be vanished or forget since there are more powerful alternatives in the kernel tree for more wider use cases.
I think there could be some example fs to show Rust VFS abstraction (such as ext2, and even minix fs). Those fses shouldn't be too hard to get a Rust implementation (e.g. minix fs for pre Linux 1.0). But honestly I don't think it's a good idea to upstream a narrow use case stuff even it's written in Rust: also considering Wedson has been quited, so the code may not be maintainerd anymore.
In short, I do _hope_ a nice Rust VFS abstraction could be landed upstream. But it should be driven by a simple no-journal read-write filesystem to check all Rust VFS components in the global vision instead of some unsustainable random upstream work just for corporation pride likewise.
And if some other approach could compare EROFS as a known prior art (as I once fully compared with SquashFS in the paper) with good reasons, I will be very happy and feel respect (also I could know the limitation of EROFS or how to improve EROFS.) But if there is no reason and just ignore EROFS exists, and I think it's not the proper way to propose a new kernel feature / filesystem.
[2] https://lwn.net/Articles/934047
[3] https://github.com/kata-containers/kata-containers/pull/7106#issuecomment-1592192981
Quoted 14 lines
Quoted 10 lines
Quoted 9 lines
Miguel Ojeda.
However, we can only reach that longterm goal if maintainers are willing and ready to put Rust into their subsystems (either by knowing/learning Rust themselves or by having a co-maintainer that does just the Rust part). So you wanting to experiment is great. I appreciate that you also have a student working on this. Still, I think we should follow our guidelines and create abstractions in order to require as littleunsafecode as possible.Quoted 3 lines
I've expressed my point. I don't think some
guidelinecould bring success to RFL. Since many subsystems needs an incremental way, not just a black-or-white thing.
Quoted 6 lines
Incremental is good, and if you want to use a .rs file or two in the middle of your module, that's fine. But please don't try to implement bindings to common kernel data structures like inodes and dentries and superblocks if at all possible and ignore the work that others are doing in this area as that's just duplicated work and will cause more confusion over time.
Yeah, agreed. That is what I'd like to say.
Honestly, Yiyang don't have enough time to implement VFS abstraction due to his studies and my time slot is limited for now too.
So I also asked him to "don't touch common kernel data structures like inodes and dentries and superblocks if possible" and just convert the EROFS core logic.
But it seems his RFC patch is still left something, I think he will address them the next version.
Quoted 6 lines
It's the same for drivers, I will object strongly if someone attempted to create a USB binding for 'struct
usb_interface' in the middle of a USB driver and instead insist they work on a generic binding that can be used by all USB drivers. I imagine the VFS maintainers have the same opinion on their apis as well for valid reasons.
Agreed.
Thanks,
Gao Xiang
Quoted 4 lines
thanks,
greg k-h
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
In reply to Benno LossinQuoted 77 lines · Benno Lossin via Linux-erofs
Quoted 31 lines · Gao Xiang
Hi Benno,
Quoted 19 lines · Benno Lossin
Hi,
Thanks for the patch series. I think it's great that you want to use Rust for this filesystem.
Quoted 6 lines · Gao Xiang
Quoted 4 lines · Gary Guo
Also, it seems that you're building abstractions into EROFS directly without building a generic abstraction. We have been avoiding that. If there's an abstraction that you need and missing, please add that abstraction. In fact, there're a bunch of people trying to add FS
Quoted 6 lines
No, I'd like to try to replace some EROFS C logic first to Rust (by using EROFS C API interfaces) and try if Rust is really useful for a real in-tree filesystem. If Rust can improve EROFS security or performance (although I'm sceptical on performance), As an EROFS maintainer, I'm totally fine to accept EROFS Rust logic landed to help the whole filesystem better.
Quoted 8 lines
As Gary already said, we have been using a different approach and it has served us well. Your approach of calling directly into C from the driver can be used to create a proof of concept, but in our opinion it is not something that should be put into mainline. That is because calling C from Rust is rather complicated due to the many nuanced features that Rust provides (for example the safety requirements of references). Therefore moving the dangerous parts into a central location is crucial for making use of all of Rust's advantages inside of your code.
Quoted 4 lines
I'm not quite sure about your point honestly. In my opinion, there is nothing different to use Rust
_withinafilesystem_or_withinadriver_or_withina Linuxsubsystem_as long as all negotiated APIs are audited.To us there is a big difference: If a lot of functions in an API are
unsafewithout being inherent from the problem that it solves, then it's a bad API.Quoted 4 lines
Otherwise, it means Rust will never be used to write Linux core parts such as MM, VFS or block layer. Does this point make sense? At least, Rust needs to get along with the existing C code (in an audited way) rather than refuse C code.
I am neither requiring you to write solely safe code, nor am I banning interacting with the C side. What we mean when we talk about abstractions is that we want to minimize the Rust code that directly interfaces with C. Rust-to-Rust interfaces can be a lot safer and are easier to implement correctly.
Quoted 3 lines
My personal idea about Rust: I think Rust is just another
_languagetool_for the Linux kernel which could save us time and make the kernel development better.Yes, but we do have conventions, rules and guidelines for writing such code. C code also has them. If you want/need to break them, there should be a good reason to do so. I don't see one in this instance.
Quoted 2 lines
Or I wonder why not writing a complete new Rust stuff instead rather than living in the C world?
There are projects that do that yes. But Rust-for-Linux is about bringing Rust to the kernel and part of that is coming up with good conventions and rules.
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
For Rust VFS abstraction, that is a different and indepenent story, Yiyang don't have any bandwidth on this due to his limited time.
This seems a bit weird, you have the bandwidth to write your own abstractions, but not use the stuff that has already been developed?
It's not written by me, Yiyang is still an undergraduate tudent. It's his research project and I don't think it's his responsibility to make an upstreamable VFS abstraction.
Quoted 7 lines
That is fair, but he wouldn't have to start from scratch, Wedsons abstractions were good enough for him to write a Rust version of ext2. In addition, tarfs and puzzlefs also use those bindings.
To me it sounds as if you have not taken the time to try to make it work with the existing abstractions. Have you tried reaching out to Ariel? He is working on puzzlefs and might have some insight to give you. Sadly Wedson has left the project, so someone will have to pick up his work.
I share the same opinions as Benno that we should try to use the existing filesystem abstractions, even if they are not yet upstream. Since erofs is a read-only filesystem and the Rust filesystem abstractions are also used by other two read-only filesystems (TarFS and PuzzleFS), it shouldn't be too difficult to adapt the erofs Rust code so that it also uses the existing filesystem abstractions. And if there is anything lacking, we can improve the existing generic APIs. This would also increase the chances of upstreaming them.
I'm happy to help you if you decide to go down this route.
Cheers,
Ariel
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Hi Ariel,
On 2024/9/25 23:48, Ariel Miculas wrote:
...
Quoted 8 lines
I share the same opinions as Benno that we should try to use the existing filesystem abstractions, even if they are not yet upstream. Since erofs is a read-only filesystem and the Rust filesystem abstractions are also used by other two read-only filesystems (TarFS and PuzzleFS), it shouldn't be too difficult to adapt the erofs Rust code so that it also uses the existing filesystem abstractions. And if there is anything lacking, we can improve the existing generic APIs. This would also increase the chances of upstreaming them.
I've expressed my ideas about "TarFS" [1] and PuzzleFS already: since I'm one of the EROFS authors, I should be responsible for this long-term project as my own promise to the Linux community and makes it serve for more Linux users (it has not been interrupted since 2017, even I sacrificed almost all my leisure time because the EROFS project isn't all my paid job, I need to maintain our internal kernel storage stack too).
[1] https://lore.kernel.org/r/[email protected]
Basically there should be some good reasons to upstream a new stuff to Linux kernel, I believe it has no exception on the Rust side even it's somewhat premature: please help compare to the prior arts in details.
And there are all thoughts for reference [2][3][4][5]:
[2] https://github.com/project-machine/puzzlefs/issues/114#issuecomment-2369872133
[3] https://github.com/opencontainers/image-spec/issues/1190#issuecomment-2138572683
[4] https://lore.kernel.org/linux-fsdevel/[email protected]/
[5] https://lore.kernel.org/linux-fsdevel/20230609-nachrangig-handwagen-375405d3b9f1@brauner/
Here still, I do really want to collaborate with you on your reasonable use cases. But if you really want to do your upstream attempt without even any comparsion, please go ahead because I believe I can only express my own opinion, but I really don't decide if your work is acceptable for the kernel.
Quoted 2 lines
I'm happy to help you if you decide to go down this route.
Again, the current VFS abstraction is totally incomplete and broken [6].
I believe it should be driven by a full-featured read-write fs [7] (even like a simple minix fs in pre-Linux 1.0 era) and EROFS will use Rust in "fs/erofs" as the experiment, but I will definitely polish the Rust version until it looks good before upstreaming.
I really don't want to be a repeater again.
[6] https://lwn.net/SubscriberLink/991062/9de8e9a466a3faf5
[7] https://lore.kernel.org/linux-fsdevel/ZZ3GeehAw%[email protected]
Thanks,
Gao Xiang
Quoted 3 lines
Cheers,
Ariel
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Quoted 35 lines · Gao Xiang
Hi Ariel,
On 2024/9/25 23:48, Ariel Miculas wrote:
...
Quoted 8 lines
I share the same opinions as Benno that we should try to use the existing filesystem abstractions, even if they are not yet upstream. Since erofs is a read-only filesystem and the Rust filesystem abstractions are also used by other two read-only filesystems (TarFS and PuzzleFS), it shouldn't be too difficult to adapt the erofs Rust code so that it also uses the existing filesystem abstractions. And if there is anything lacking, we can improve the existing generic APIs. This would also increase the chances of upstreaming them.
I've expressed my ideas about "TarFS" [1] and PuzzleFS already: since I'm one of the EROFS authors, I should be responsible for this long-term project as my own promise to the Linux community and makes it serve for more Linux users (it has not been interrupted since 2017, even I sacrificed almost all my leisure time because the EROFS project isn't all my paid job, I need to maintain our internal kernel storage stack too).
[1] https://lore.kernel.org/r/[email protected]
Basically there should be some good reasons to upstream a new stuff to Linux kernel, I believe it has no exception on the Rust side even it's somewhat premature: please help compare to the prior arts in details.
And there are all thoughts for reference [2][3][4][5]:
[2] https://github.com/project-machine/puzzlefs/issues/114#issuecomment-2369872133
[3] https://github.com/opencontainers/image-spec/issues/1190#issuecomment-2138572683
[4] https://lore.kernel.org/linux-fsdevel/[email protected]/
[5] https://lore.kernel.org/linux-fsdevel/20230609-nachrangig-handwagen-375405d3b9f1@brauner/
Quoted 6 lines
Here still, I do really want to collaborate with you on your reasonable use cases. But if you really want to do your upstream attempt without even any comparsion, please go ahead because I believe I can only express my own opinion, but I really don't decide if your work is acceptable for the kernel.
Thanks for your thoughts on PuzzleFS, I would really like if we could centralize the discussions on the latest patch series I sent to the mailing lists back in May [1]. The reason I say this is because looking at that thread, it seems there is no feedback for PuzzleFS. The feedback exists, it's just scattered throughout different mediums. On top of this, I would also like to engage in the discussions with Dave Chinner, so I can better understand the limitations of PuzzleFS and the reasons for which it might be rejected in the Linux Kernel. I do appreciate your feedback and I need to take my time to respond to the technical issues that you brought up in the github issue.
However, even if it's not upstream, PuzzleFS does use the latest Rust filesystem abstractions and thus it stands as an example of how to use them. And this thread is not about PuzzleFS, but about the Rust filesystem abstractions and how one might start to use them. That's where I offered to help, since I already went through the process of having to use them.
[1] https://lore.kernel.org/all/[email protected]/
Quoted 5 lines
Quoted 2 lines
I'm happy to help you if you decide to go down this route.
Again, the current VFS abstraction is totally incomplete and broken [6].
If they're incomplete, we can work together to implement the missing functionalities. Furthermore, we can work to fix the broken stuff. I don't think these are good reasons to completely ignore the work that's already been done on this topic.
By the way, what is it that's actually broken? You've linked to an LWN article [2] (or at least I think your 6th link was supposed to link to "Rust for filesystems" instead of the "Committing to Rust in the kernel" one), but I'm interested in the specifics. What exactly doesn't work as expected from the filesystem abstractions?
[2] https://lwn.net/Articles/978738/
Quoted 3 lines
I believe it should be driven by a full-featured read-write fs [7] (even like a simple minix fs in pre-Linux 1.0 era) and EROFS will
I do find it weird that you want a full-featured read-write fs implemented in Rust, when erofs is a read-only filesystem.
Quoted 2 lines
use Rust in "fs/erofs" as the experiment, but I will definitely polish the Rust version until it looks good before upstreaming.
I honestly don't see how it would look good if they're not using the existing filesystem abstractions. And I'm not convinced that Rust in the kernel would be useful in any way without the many subsystem abstractions which were implemented by the Rust for Linux team for the past few years.
Cheers,
Ariel
Quoted 13 lines
I really don't want to be a repeater again.
[6] https://lwn.net/SubscriberLink/991062/9de8e9a466a3faf5
[7] https://lore.kernel.org/linux-fsdevel/ZZ3GeehAw%[email protected]Thanks,
Gao XiangQuoted 3 lines
Cheers,
Ariel
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Quoted 5 lines · Ariel Miculas
Quoted 4 lines · Gao Xiang
Hi Ariel,
On 2024/9/25 23:48, Ariel Miculas wrote:
...
Quoted 14 lines
Quoted 7 lines
And there are all thoughts for reference [2][3][4][5]:
[2] https://github.com/project-machine/puzzlefs/issues/114#issuecomment-2369872133
[3] https://github.com/opencontainers/image-spec/issues/1190#issuecomment-2138572683
[4] https://lore.kernel.org/linux-fsdevel/[email protected]/
[5] https://lore.kernel.org/linux-fsdevel/20230609-nachrangig-handwagen-375405d3b9f1@brauner/Quoted 6 lines
Here still, I do really want to collaborate with you on your reasonable use cases. But if you really want to do your upstream attempt without even any comparsion, please go ahead because I believe I can only express my own opinion, but I really don't decide if your work is acceptable for the kernel.
Quoted 10 lines
Thanks for your thoughts on PuzzleFS, I would really like if we could centralize the discussions on the latest patch series I sent to the mailing lists back in May [1]. The reason I say this is because looking at that thread, it seems there is no feedback for PuzzleFS. The feedback exists, it's just scattered throughout different mediums. On top of this, I would also like to engage in the discussions with Dave Chinner, so I can better understand the limitations of PuzzleFS and the reasons for which it might be rejected in the Linux Kernel. I do appreciate your feedback and I need to take my time to respond to the technical issues that you brought up in the github issue.
In short, I really want to avoid open arbitary number files in the page fault path regardless of the performance concerns, because even there are many cases that mmap_lock is dropped, but IMHO there is still cases that mmap_lock will be taken.
IOWs, I think it's controversal for a kernel fs to open random file in the page fault context under mmap_lock in the begining. Otherwise, it's pretty straight-forward to add some similiar feature to EROFS.
Quoted 13 lines
However, even if it's not upstream, PuzzleFS does use the latest Rust filesystem abstractions and thus it stands as an example of how to use them. And this thread is not about PuzzleFS, but about the Rust filesystem abstractions and how one might start to use them. That's where I offered to help, since I already went through the process of having to use them.
[1] https://lore.kernel.org/all/[email protected]/
Quoted 3 lines
Quoted 2 lines
I'm happy to help you if you decide to go down this route.
Quoted 7 lines
Quoted 2 lines
Again, the current VFS abstraction is totally incomplete and broken [6].
If they're incomplete, we can work together to implement the missing functionalities. Furthermore, we can work to fix the broken stuff. I don't think these are good reasons to completely ignore the work that's already been done on this topic.
I've said, we don't miss any Rust VFS abstraction work, as long as some work lands in the Linux kernel, we will switch to use them.
The reason we don't do that is again
- I don't have time to work on this because my life is still limited for RFL in any way at least this year; I don't know if Yiyang has time to work on a complete ext2 and a Rust VFS abstraction.
- We just would like to _use Rust_ for the core EROFS logic, instead of touching any VFS stuff. I'm not sure why it's called "completely ignore the VFS abstraction", because there is absolutely no relationship between these two things. Why we need to mix them up?
Quoted 6 lines
By the way, what is it that's actually broken? You've linked to an LWN article [2] (or at least I think your 6th link was supposed to link to "Rust for filesystems" instead of the "Committing to Rust in the kernel" one), but I'm interested in the specifics. What exactly doesn't work as expected from the filesystem abstractions?
For example, with my current Rust skill, I'm not sure why fill_super for "T::SUPER_TYPE, sb::Type::BlockDev" must use "new_sb.bdev().inode().mapper()".
It's unnecessary for a bdev-based fs to use bdev inode page cache to read metadata;
Also it's unnecessary for a const fs type to besb::Type::BlockDev or sb::Type::Independent as
/// Determines how superblocks for this file system type are keyed.
+ const SUPER_TYPE: sb::Type = sb::Type::Independent;
because at least for the current EROFS use cases, we will decide to use get_tree_bdev() or get_tree_nodev() according to the mount source or mount options.
Quoted 9 lines
[2] https://lwn.net/Articles/978738/
Quoted 3 lines
I believe it should be driven by a full-featured read-write fs [7] (even like a simple minix fs in pre-Linux 1.0 era) and EROFS will
I do find it weird that you want a full-featured read-write fs implemented in Rust, when erofs is a read-only filesystem.
I'm not sure why it's weird from the sane Rust VFS abstraction perspective.
Quoted 9 lines
Quoted 3 lines
use Rust in "fs/erofs" as the experiment, but I will definitely polish the Rust version until it looks good before upstreaming.
I honestly don't see how it would look good if they're not using the existing filesystem abstractions. And I'm not convinced that Rust in the kernel would be useful in any way without the many subsystem abstractions which were implemented by the Rust for Linux team for the past few years.
So let's see the next version.
Thanks,
Gao Xiang
Quoted 4 lines
Cheers,
Ariel
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Quoted 3 lines · Gao Xiang
On 2024/9/26 05:45, Ariel Miculas wrote:
...
Quoted 8 lines
Quoted 6 lines
I honestly don't see how it would look good if they're not using the existing filesystem abstractions. And I'm not convinced that Rust in the kernel would be useful in any way without the many subsystem abstractions which were implemented by the Rust for Linux team for the past few years.
So let's see the next version.
Some more words, regardless of in-tree "fs/xfs/libxfs",
you also claimed "Another goal is to share the same code between user space and kernel space in order to provide one secure implementation." for example in [1].
I wonder Rust kernel VFS abstraction is forcely used in your userspace implementation, or (somewhat) your argument is still broken here.
[1] https://lore.kernel.org/r/20230609-feldversuch-fixieren-fa141a2d9694@brauner
Thanks,
Gao Xiang
Quoted 8 lines
Thanks,
Gao XiangQuoted 4 lines
Cheers,
Ariel
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Quoted 18 lines · Gao Xiang
Quoted 3 lines · Gao Xiang
On 2024/9/26 05:45, Ariel Miculas wrote:
...
Quoted 8 lines
Quoted 6 lines
I honestly don't see how it would look good if they're not using the existing filesystem abstractions. And I'm not convinced that Rust in the kernel would be useful in any way without the many subsystem abstractions which were implemented by the Rust for Linux team for the past few years.
So let's see the next version.
Quoted 7 lines
Some more words, regardless of in-tree "
fs/xfs/libxfs",
you also claimed "Another goal is to share the same code between user space and kernel space in order to provide one secure implementation." for example in [1].I wonder Rust kernel VFS abstraction is forcely used in your userspace implementation, or (somewhat) your argument is still broken here.
Of course the implementations cannot be identical, but there is a lot of shared code between the user space and kernel space PuzzleFS implementations. The user space implementation uses the fuser [1] crate and implicitly its API for implementing the read/seek/list_xattrs etc. operations, while the kernel implementation uses the Rust filesystem abstractions.
While it's currently not possible to use external crates in the Linux kernel (maybe it won't ever be), one area for improvement would be to keep the shared code between these PuzzleFS implementations in the kernel and publish releases to crates.io from there. In this way it will be obvious which parts of the code are shared and they will actually be shared (right now the code is duplicated).
I've actually touched on these points [2] during my last year's presentation of PuzzleFS at Open Source Summit Europe [3].
And here [4] you can see the space savings achieved by PuzzleFS. In short, if you take 10 versions of Ubuntu Jammy from dockerhub, they take up 282 MB. Convert them to PuzzleFS and they only take up 130 MB (this is before applying any compression, the space savings are only due to the chunking algorithm). If we enable compression (PuzzleFS uses Zstd seekable compression), which is a fairer comparison (considering that the OCI image uses gzip compression), then we get down to 53 MB for storing all 10 Ubuntu Jammy versions using PuzzleFS.
Here's a summary:
# Steps
* I’ve downloaded 10 versions of Jammy from hub.docker.com
* These images only have one layer which is in tar.gz format
* I’ve built 10 equivalent puzzlefs images
* Compute the tarball_total_size by summing the sizes of every Jammy tarball (uncompressed) => 766 MB (use this as baseline)
* Sum the sizes of every oci/puzzlefs image => total_size
* Compute the total size as if all the versions were stored in a single oci/puzzlefs repository => total_unified_size
* Saved space = tarball_total_size - total_unified_size
# Results
(See [5] if you prefer the video format)
| Type | Total size (MB) | Average layer size (MB) | Unified size (MB) | Saved (MB) / 766 MB | | --- | --- | --- | --- | --- | | Oci (uncompressed) | 766 | 77 | 766 | 0 (0%) | | PuzzleFS uncompressed | 748 | 74 | 130 | 635 (83%) | | Oci (compressed) | 282 | 28 | 282 | 484 (63%) | | PuzzleFS (compressed) | 298 | 30 | 53 | 713 (93%) |
Here's the script I used to download the Ubuntu Jammy versions and generate the PuzzleFS images [6] to get an idea about how I got to these results.
Can we achieve these results with the current erofs features? I'm referring specifically to this comment: "EROFS already supports variable-sized chunks + CDC" [7].
[1] https://docs.rs/fuser/latest/fuser/
[2] https://youtu.be/OhMtoLrjiBY?si=iuk7PstznEUgnr4g&t=1150
[3] https://osseu2023.sched.com/event/1OGjk/puzzlefs-the-next-generation-container-filesystem-armand-ariel-miculas-cisco-systems
[4] https://youtu.be/OhMtoLrjiBY?si=jwReE1qjs1wXLUCr&t=1732
[5] https://youtu.be/OhMtoLrjiBY?si=Nhlz8FJ9CGnwgOlS&t=1862
[6] https://gist.github.com/ariel-miculas/956056f213db2d3027905c61264d160b
[7] https://github.com/project-machine/puzzlefs/issues/114#issuecomment-2367039464
Regards,
Ariel
Quoted 15 lines
[1] https://lore.kernel.org/r/20230609-feldversuch-fixieren-fa141a2d9694@brauner
Thanks,
Gao XiangQuoted 8 lines
Thanks,
Gao XiangQuoted 4 lines
Cheers,
Ariel
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Quoted 2 lines · Ariel Miculas
Quoted 1 line · Gao Xiang
...
Quoted 33 lines
And here [4] you can see the space savings achieved by PuzzleFS. In short, if you take 10 versions of Ubuntu Jammy from dockerhub, they take up 282 MB. Convert them to PuzzleFS and they only take up 130 MB (this is before applying any compression, the space savings are only due to the chunking algorithm). If we enable compression (PuzzleFS uses Zstd seekable compression), which is a fairer comparison (considering that the OCI image uses gzip compression), then we get down to 53 MB for storing all 10 Ubuntu Jammy versions using PuzzleFS.
Here's a summary:
# Steps* I’ve downloaded 10 versions of Jammy from hub.docker.com
* These images only have one layer which is in tar.gz format
* I’ve built 10 equivalent puzzlefs images
* Compute thetarball_total_sizeby summing the sizes of every Jammy tarball (uncompressed) => 766 MB (use this as baseline)
* Sum the sizes of every oci/puzzlefs image =>total_size
* Compute the total size as if all the versions were stored in a single oci/puzzlefs repository =>total_unified_size
* Saved space =tarball_total_size-total_unified_size# Results
(See [5] if you prefer the video format)| Type | Total size (MB) | Average layer size (MB) | Unified size (MB) | Saved (MB) / 766 MB | | --- | --- | --- | --- | --- | | Oci (uncompressed) | 766 | 77 | 766 | 0 (0%) | | PuzzleFS uncompressed | 748 | 74 | 130 | 635 (83%) | | Oci (compressed) | 282 | 28 | 282 | 484 (63%) | | PuzzleFS (compressed) | 298 | 30 | 53 | 713 (93%) |
Quoted 7 lines
Here's the script I used to download the Ubuntu Jammy versions and generate the PuzzleFS images [6] to get an idea about how I got to these results.
Can we achieve these results with the current erofs features? I'm referring specifically to this comment: "EROFS already supports variable-sized chunks + CDC" [7].
Please see
https://erofs.docs.kernel.org/en/latest/comparsion/dedupe.html
Total Size (MiB) Average layer size (MiB) Saved / 766.1MiB
Compressed OCI (tar.gz) 282.5 28.3 63%
Uncompressed OCI (tar) 766.1 76.6 0%
Uncomprssed EROFS 109.5 11.0 86%
EROFS (DEFLATE,9,32k) 46.4 4.6 94%
EROFS (LZ4HC,12,64k) 54.2 5.4 93%
I don't know which compression algorithm are you using (maybe Zstd?), but from the result is
EROFS (LZ4HC,12,64k) 54.2
PuzzleFS compressed 53?
EROFS (DEFLATE,9,32k) 46.4
I could reran with EROFS + Zstd, but it should be smaller. This feature has been supported since Linux 6.1, thanks.
Thanks,
Gao Xiang
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Quoted 46 lines · Gao Xiang
Quoted 2 lines · Ariel Miculas
Quoted 1 line · Gao Xiang
...
Quoted 37 lines
And here [4] you can see the space savings achieved by PuzzleFS. In short, if you take 10 versions of Ubuntu Jammy from dockerhub, they take up 282 MB. Convert them to PuzzleFS and they only take up 130 MB (this is before applying any compression, the space savings are only due to the chunking algorithm). If we enable compression (PuzzleFS uses Zstd seekable compression), which is a fairer comparison (considering that the OCI image uses gzip compression), then we get down to 53 MB for storing all 10 Ubuntu Jammy versions using PuzzleFS.
Here's a summary:
# Steps* I’ve downloaded 10 versions of Jammy from hub.docker.com
* These images only have one layer which is in tar.gz format
* I’ve built 10 equivalent puzzlefs images
* Compute thetarball_total_sizeby summing the sizes of every Jammy tarball (uncompressed) => 766 MB (use this as baseline)
* Sum the sizes of every oci/puzzlefs image =>total_size
* Compute the total size as if all the versions were stored in a single oci/puzzlefs repository =>total_unified_size
* Saved space =tarball_total_size-total_unified_size# Results
(See [5] if you prefer the video format)| Type | Total size (MB) | Average layer size (MB) | Unified size (MB) | Saved (MB) / 766 MB | | --- | --- | --- | --- | --- | | Oci (uncompressed) | 766 | 77 | 766 | 0 (0%) | | PuzzleFS uncompressed | 748 | 74 | 130 | 635 (83%) | | Oci (compressed) | 282 | 28 | 282 | 484 (63%) | | PuzzleFS (compressed) | 298 | 30 | 53 | 713 (93%) |Here's the script I used to download the Ubuntu Jammy versions and generate the PuzzleFS images [6] to get an idea about how I got to these results.
Quoted 6 lines
Quoted 3 lines
Can we achieve these results with the current erofs features? I'm referring specifically to this comment: "EROFS already supports variable-sized chunks + CDC" [7].
Please see
https://erofs.docs.kernel.org/en/latest/comparsion/dedupe.html
Great, I see you've used the same example as I did. Though I must admit I'm a little surprised there's no mention of PuzzleFS in your document.
Quoted 8 lines
Total Size (MiB) Average layer size (MiB) Saved / 766.1MiB
Compressed OCI (tar.gz) 282.5 28.3 63%
Uncompressed OCI (tar) 766.1 76.6 0%
Uncomprssed EROFS 109.5 11.0 86%
EROFS (DEFLATE,9,32k) 46.4 4.6 94%
EROFS (LZ4HC,12,64k) 54.2 5.4 93%
Quoted 8 lines
I don't know which compression algorithm are you using (maybe Zstd?), but from the result is
EROFS (LZ4HC,12,64k) 54.2
PuzzleFS compressed 53?
EROFS (DEFLATE,9,32k) 46.4I could reran with EROFS + Zstd, but it should be smaller. This feature has been supported since Linux 6.1, thanks.
The average layer size is very impressive for EROFS, great work. However, if we multiply the average layer size by 10, we get the total size (5.4 MiB * 10 ~ 54.2 MiB), whereas for PuzzleFS, we see that while the average layer size is 30 MIB (for the compressed case), the unified size is only 53 MiB. So this tells me there's blob sharing between the different versions of Ubuntu Jammy with PuzzleFS, but there's no sharing with EROFS (what I'm talking about is deduplication across the multiple versions of Ubuntu Jammy and not within one single version).
Of course, with only 10 images, the space savings don't seem that impressive for PuzzleFS compared to EROFS, but imagine we are storing hundreds/thousands of Ubuntu versions. Then we're also building OCI images on top of these versions. So if the user already has all the blobs for an Ubuntu version, then we only need to ship the chunks that have changed / have been added as a result of the specific application that we've built on top of an existing Ubuntu version.
One more thing: the "Unified size" column is the key for understanding the space savings offered by PuzzleFS and I see that you've left this column out of your table.
Regards,
Ariel
Quoted 3 lines
Thanks,
Gao Xiang
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Quoted 51 lines · Ariel Miculas
Quoted 50 lines · Gao Xiang
Quoted 2 lines · Ariel Miculas
Quoted 1 line · Gao Xiang
...
Quoted 40 lines
And here [4] you can see the space savings achieved by PuzzleFS. In short, if you take 10 versions of Ubuntu Jammy from dockerhub, they take up 282 MB. Convert them to PuzzleFS and they only take up 130 MB (this is before applying any compression, the space savings are only due to the chunking algorithm). If we enable compression (PuzzleFS uses Zstd seekable compression), which is a fairer comparison (considering that the OCI image uses gzip compression), then we get down to 53 MB for storing all 10 Ubuntu Jammy versions using PuzzleFS.
Here's a summary:
# Steps* I’ve downloaded 10 versions of Jammy from hub.docker.com
* These images only have one layer which is in tar.gz format
* I’ve built 10 equivalent puzzlefs images
* Compute thetarball_total_sizeby summing the sizes of every Jammy tarball (uncompressed) => 766 MB (use this as baseline)
* Sum the sizes of every oci/puzzlefs image =>total_size
* Compute the total size as if all the versions were stored in a single oci/puzzlefs repository =>total_unified_size
* Saved space =tarball_total_size-total_unified_size# Results
(See [5] if you prefer the video format)| Type | Total size (MB) | Average layer size (MB) | Unified size (MB) | Saved (MB) / 766 MB | | --- | --- | --- | --- | --- | | Oci (uncompressed) | 766 | 77 | 766 | 0 (0%) | | PuzzleFS uncompressed | 748 | 74 | 130 | 635 (83%) | | Oci (compressed) | 282 | 28 | 282 | 484 (63%) | | PuzzleFS (compressed) | 298 | 30 | 53 | 713 (93%) |Here's the script I used to download the Ubuntu Jammy versions and generate the PuzzleFS images [6] to get an idea about how I got to these results.
Can we achieve these results with the current erofs features? I'm referring specifically to this comment: "EROFS already supports variable-sized chunks + CDC" [7].
Quoted 5 lines
Quoted 2 lines
Please see
https://erofs.docs.kernel.org/en/latest/comparsion/dedupe.htmlGreat, I see you've used the same example as I did. Though I must admit I'm a little surprised there's no mention of PuzzleFS in your document.
Why I need to mention and even try PuzzleFS here (there are too many attempts why I need to try them all)? It just compares to the EROFS prior work.
Quoted 18 lines
Quoted 9 lines
Total Size (MiB) Average layer size (MiB) Saved / 766.1MiB
Compressed OCI (tar.gz) 282.5 28.3 63%
Uncompressed OCI (tar) 766.1 76.6 0%
Uncomprssed EROFS 109.5 11.0 86%
EROFS (DEFLATE,9,32k) 46.4 4.6 94%
EROFS (LZ4HC,12,64k) 54.2 5.4 93%Quoted 8 lines
I don't know which compression algorithm are you using (maybe Zstd?), but from the result is
EROFS (LZ4HC,12,64k) 54.2
PuzzleFS compressed 53?
EROFS (DEFLATE,9,32k) 46.4I could reran with EROFS + Zstd, but it should be smaller. This feature has been supported since Linux 6.1, thanks.
Quoted 8 lines
The average layer size is very impressive for EROFS, great work. However, if we multiply the average layer size by 10, we get the total size (5.4 MiB * 10 ~ 54.2 MiB), whereas for PuzzleFS, we see that while the average layer size is 30 MIB (for the compressed case), the unified size is only 53 MiB. So this tells me there's blob sharing between the different versions of Ubuntu Jammy with PuzzleFS, but there's no sharing with EROFS (what I'm talking about is deduplication across the multiple versions of Ubuntu Jammy and not within one single version).
Don't make me wrong, I don't think you got the point.
First, what you asked was `I'm referring specifically to this
comment: "EROFS already supports variable-sized chunks + CDC"`, so I clearly answered with the result of compressed data global deduplication with CDC.
Here both EROFS and Squashfs compresses 10 Ubuntu images into one image for fair comparsion to show the benefit of CDC, so I believe they basically equal to your Unified sizes, so the result is
Your unified size EROFS (LZ4HC,12,64k) 54.2 PuzzleFS compressed 53? EROFS (DEFLATE,9,32k) 46.4
That is why I used your 53 unified size to show EROFS is much smaller than PuzzleFS.
The reason why EROFS and SquashFS doesn't have the Total Sizes is just because we cannot store every individual chunk into some seperate file.
Currently, I have seen no reason to open arbitary kernel files (maybe hundreds due to large folio feature at once) in the page fault context. If I modified mkfs.erofs tool, I could give some similar numbers, but I don't want to waste time now due to open arbitary kernel files in the page fault context.
As I said, if PuzzleFS finally upstream some work to open kernel files in page fault context, I will definitely work out the same feature for EROFS soon, but currently I don't do that just because it's very controversal and no in-tree kernel filesystem does that.
Thanks,
Gao Xiang
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Quoted 93 lines · Gao Xiang
Quoted 51 lines · Ariel Miculas
Quoted 50 lines · Gao Xiang
Quoted 2 lines · Ariel Miculas
Quoted 1 line · Gao Xiang
...
Quoted 40 lines
And here [4] you can see the space savings achieved by PuzzleFS. In short, if you take 10 versions of Ubuntu Jammy from dockerhub, they take up 282 MB. Convert them to PuzzleFS and they only take up 130 MB (this is before applying any compression, the space savings are only due to the chunking algorithm). If we enable compression (PuzzleFS uses Zstd seekable compression), which is a fairer comparison (considering that the OCI image uses gzip compression), then we get down to 53 MB for storing all 10 Ubuntu Jammy versions using PuzzleFS.
Here's a summary:
# Steps* I’ve downloaded 10 versions of Jammy from hub.docker.com
* These images only have one layer which is in tar.gz format
* I’ve built 10 equivalent puzzlefs images
* Compute thetarball_total_sizeby summing the sizes of every Jammy tarball (uncompressed) => 766 MB (use this as baseline)
* Sum the sizes of every oci/puzzlefs image =>total_size
* Compute the total size as if all the versions were stored in a single oci/puzzlefs repository =>total_unified_size
* Saved space =tarball_total_size-total_unified_size# Results
(See [5] if you prefer the video format)| Type | Total size (MB) | Average layer size (MB) | Unified size (MB) | Saved (MB) / 766 MB | | --- | --- | --- | --- | --- | | Oci (uncompressed) | 766 | 77 | 766 | 0 (0%) | | PuzzleFS uncompressed | 748 | 74 | 130 | 635 (83%) | | Oci (compressed) | 282 | 28 | 282 | 484 (63%) | | PuzzleFS (compressed) | 298 | 30 | 53 | 713 (93%) |Here's the script I used to download the Ubuntu Jammy versions and generate the PuzzleFS images [6] to get an idea about how I got to these results.
Can we achieve these results with the current erofs features? I'm referring specifically to this comment: "EROFS already supports variable-sized chunks + CDC" [7].
Quoted 5 lines
Quoted 2 lines
Please see
https://erofs.docs.kernel.org/en/latest/comparsion/dedupe.htmlGreat, I see you've used the same example as I did. Though I must admit I'm a little surprised there's no mention of PuzzleFS in your document.
Why I need to mention and even try PuzzleFS here (there are too many attempts why I need to try them all)? It just compares to the EROFS prior work.
Quoted 18 lines
Quoted 9 lines
Total Size (MiB) Average layer size (MiB) Saved / 766.1MiB
Compressed OCI (tar.gz) 282.5 28.3 63%
Uncompressed OCI (tar) 766.1 76.6 0%
Uncomprssed EROFS 109.5 11.0 86%
EROFS (DEFLATE,9,32k) 46.4 4.6 94%
EROFS (LZ4HC,12,64k) 54.2 5.4 93%Quoted 8 lines
I don't know which compression algorithm are you using (maybe Zstd?), but from the result is
EROFS (LZ4HC,12,64k) 54.2
PuzzleFS compressed 53?
EROFS (DEFLATE,9,32k) 46.4I could reran with EROFS + Zstd, but it should be smaller. This feature has been supported since Linux 6.1, thanks.
Quoted 8 lines
The average layer size is very impressive for EROFS, great work. However, if we multiply the average layer size by 10, we get the total size (5.4 MiB * 10 ~ 54.2 MiB), whereas for PuzzleFS, we see that while the average layer size is 30 MIB (for the compressed case), the unified size is only 53 MiB. So this tells me there's blob sharing between the different versions of Ubuntu Jammy with PuzzleFS, but there's no sharing with EROFS (what I'm talking about is deduplication across the multiple versions of Ubuntu Jammy and not within one single version).
Don't make me wrong, I don't think you got the point.
Quoted 7 lines
First, what you asked was `I'm referring specifically to this
comment: "EROFS already supports variable-sized chunks + CDC"`, so I clearly answered with the result of compressed data global deduplication with CDC.Here both EROFS and Squashfs compresses 10 Ubuntu images into one image for fair comparsion to show the benefit of CDC, so
It might be a fair comparison, but that's not how container images are distributed. You're trying to argue that I should just use EROFS and I'm showing you that EROFS doesn't currently support the functionality provided by PuzzleFS: the deduplication across multiple images.
Quoted 8 lines
I believe they basically equal to your
Unified sizes, so the result isYour unified size EROFS (LZ4HC,12,64k) 54.2 PuzzleFS compressed 53? EROFS (DEFLATE,9,32k) 46.4
Quoted 6 lines
That is why I used your 53 unified size to show EROFS is much smaller than PuzzleFS.
The reason why EROFS and SquashFS doesn't have the
Total Sizes is just because we cannot store every individual chunk into some seperate file.
Well storing individual chunks into separate files is the entire point of PuzzleFS.
Quoted 7 lines
Currently, I have seen no reason to open arbitary kernel files (maybe hundreds due to large folio feature at once) in the page fault context. If I modified
mkfs.erofstool, I could give some similar numbers, but I don't want to waste time now due toopen arbitary kernel files in the page fault context.
Quoted 5 lines
As I said, if PuzzleFS finally upstream some work to open kernel files in page fault context, I will definitely work out the same feature for EROFS soon, but currently I don't do that just because it's very controversal and no in-tree kernel filesystem does that.
The PuzzleFS kernel filesystem driver is still in an early POC stage, so there's still a lot more work to be done.
Regards,
Ariel
Quoted 3 lines
Thanks,
Gao Xiang
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
On 2024/9/26 19:01, Ariel Miculas wrote:
..
Quoted 26 lines
Quoted 9 lines
I believe they basically equal to your
Unified sizes, so the result isYour unified size EROFS (LZ4HC,12,64k) 54.2 PuzzleFS compressed 53? EROFS (DEFLATE,9,32k) 46.4Quoted 6 lines
That is why I used your 53 unified size to show EROFS is much smaller than PuzzleFS.
The reason why EROFS and SquashFS doesn't have the
Total Sizes is just because we cannot store every individual chunk into some seperate file.Well storing individual chunks into separate files is the entire point of PuzzleFS.
Quoted 7 lines
Currently, I have seen no reason to open arbitary kernel files (maybe hundreds due to large folio feature at once) in the page fault context. If I modified
mkfs.erofstool, I could give some similar numbers, but I don't want to waste time now due toopen arbitary kernel files in the page fault context.
Quoted 8 lines
Quoted 5 lines
As I said, if PuzzleFS finally upstream some work to open kernel files in page fault context, I will definitely work out the same feature for EROFS soon, but currently I don't do that just because it's very controversal and no in-tree kernel filesystem does that.
The PuzzleFS kernel filesystem driver is still in an early POC stage, so there's still a lot more work to be done.
I suggest that you could just ask FS/MM folks about this ("open kernel files when reading in the page fault") first.
If they say "no", I suggest please don't waste on this anymore.
Thanks,
Gao Xiang
Quoted 7 lines
Regards,
ArielQuoted 3 lines
Thanks,
Gao Xiang
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
In reply to Ariel MiculasQuoted 1 line · Ariel Miculas via Linux-erofs
On 24/09/26 06:46, Gao Xiang wrote:
...
Quoted 35 lines
Quoted 35 lines
Quoted 19 lines
Quoted 10 lines
Total Size (MiB) Average layer size (MiB) Saved / 766.1MiB
Compressed OCI (tar.gz) 282.5 28.3 63%
Uncompressed OCI (tar) 766.1 76.6 0%
Uncomprssed EROFS 109.5 11.0 86%
EROFS (DEFLATE,9,32k) 46.4 4.6 94%
EROFS (LZ4HC,12,64k) 54.2 5.4 93%Quoted 8 lines
I don't know which compression algorithm are you using (maybe Zstd?), but from the result is
EROFS (LZ4HC,12,64k) 54.2 PuzzleFS compressed 53? EROFS (DEFLATE,9,32k) 46.4I could reran with EROFS + Zstd, but it should be smaller. This feature has been supported since Linux 6.1, thanks.
Quoted 8 lines
The average layer size is very impressive for EROFS, great work. However, if we multiply the average layer size by 10, we get the total size (5.4 MiB * 10 ~ 54.2 MiB), whereas for PuzzleFS, we see that while the average layer size is 30 MIB (for the compressed case), the unified size is only 53 MiB. So this tells me there's blob sharing between the different versions of Ubuntu Jammy with PuzzleFS, but there's no sharing with EROFS (what I'm talking about is deduplication across the multiple versions of Ubuntu Jammy and not within one single version).
Don't make me wrong, I don't think you got the point.
First, what you asked was `I'm referring specifically to this
comment: "EROFS already supports variable-sized chunks + CDC"`, so I clearly answered with the result of compressed data global deduplication with CDC.
Quoted 5 lines
Quoted 2 lines
Here both EROFS and Squashfs compresses 10 Ubuntu images into one image for fair comparsion to show the benefit of CDC, so
It might be a fair comparison, but that's not how container images are distributed. You're trying to argue that I should just use EROFS and I'm
First, OCI layer is just distributed like what I said.
For example, I could introduce some common blobs to keep
chunks as chunk dictionary. And then the each image
will be just some index, and all data will be deduplicated. That is also what Nydus works.
Quoted 2 lines
showing you that EROFS doesn't currently support the functionality provided by PuzzleFS: the deduplication across multiple images.
No, EROFS supports external devices/blobs to keep a lot of chunks too (as dictionary to share data among images), but clearly it has the upper limit.
But PuzzleFS just treat each individual chunk as a seperate file, that will cause unavoidable "open arbitary number of files on reading, even in page fault context".
Quoted 26 lines
Quoted 9 lines
I believe they basically equal to your
Unified sizes, so the result isYour unified size EROFS (LZ4HC,12,64k) 54.2 PuzzleFS compressed 53? EROFS (DEFLATE,9,32k) 46.4Quoted 6 lines
That is why I used your 53 unified size to show EROFS is much smaller than PuzzleFS.
The reason why EROFS and SquashFS doesn't have the
Total Sizes is just because we cannot store every individual chunk into some seperate file.Well storing individual chunks into separate files is the entire point of PuzzleFS.
Quoted 7 lines
Currently, I have seen no reason to open arbitary kernel files (maybe hundreds due to large folio feature at once) in the page fault context. If I modified
mkfs.erofstool, I could give some similar numbers, but I don't want to waste time now due toopen arbitary kernel files in the page fault context.
Quoted 8 lines
Quoted 5 lines
As I said, if PuzzleFS finally upstream some work to open kernel files in page fault context, I will definitely work out the same feature for EROFS soon, but currently I don't do that just because it's very controversal and no in-tree kernel filesystem does that.
The PuzzleFS kernel filesystem driver is still in an early POC stage, so there's still a lot more work to be done.
I suggest that you could just ask FS/MM folks about this ("open kernel files when reading in the page fault") first.
If they say "no", I suggest please don't waste on this anymore.
Thanks,
Gao Xiang
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Quoted 48 lines · Gao Xiang
Quoted 1 line · Ariel Miculas via Linux-erofs
On 24/09/26 06:46, Gao Xiang wrote:
...
Quoted 35 lines
Quoted 35 lines
Quoted 19 lines
Quoted 10 lines
Total Size (MiB) Average layer size (MiB) Saved / 766.1MiB
Compressed OCI (tar.gz) 282.5 28.3 63%
Uncompressed OCI (tar) 766.1 76.6 0%
Uncomprssed EROFS 109.5 11.0 86%
EROFS (DEFLATE,9,32k) 46.4 4.6 94%
EROFS (LZ4HC,12,64k) 54.2 5.4 93%Quoted 8 lines
I don't know which compression algorithm are you using (maybe Zstd?), but from the result is
EROFS (LZ4HC,12,64k) 54.2 PuzzleFS compressed 53? EROFS (DEFLATE,9,32k) 46.4I could reran with EROFS + Zstd, but it should be smaller. This feature has been supported since Linux 6.1, thanks.
Quoted 8 lines
The average layer size is very impressive for EROFS, great work. However, if we multiply the average layer size by 10, we get the total size (5.4 MiB * 10 ~ 54.2 MiB), whereas for PuzzleFS, we see that while the average layer size is 30 MIB (for the compressed case), the unified size is only 53 MiB. So this tells me there's blob sharing between the different versions of Ubuntu Jammy with PuzzleFS, but there's no sharing with EROFS (what I'm talking about is deduplication across the multiple versions of Ubuntu Jammy and not within one single version).
Don't make me wrong, I don't think you got the point.
First, what you asked was `I'm referring specifically to this
comment: "EROFS already supports variable-sized chunks + CDC"`, so I clearly answered with the result of compressed data global deduplication with CDC.Quoted 5 lines
Quoted 2 lines
Here both EROFS and Squashfs compresses 10 Ubuntu images into one image for fair comparsion to show the benefit of CDC, so
It might be a fair comparison, but that's not how container images are distributed. You're trying to argue that I should just use EROFS and I'm
Quoted 6 lines
First, OCI layer is just distributed like what I said.
For example, I could introduce some common blobs to keep
chunks as chunk dictionary. And then the each image
will be just some index, and all data will be deduplicated. That is also what Nydus works.
I don't really follow what Nydus does. Here [1] it says they're using fixed size chunks of 1 MB. Where is the CDC step exactly?
[1] https://github.com/dragonflyoss/nydus/blob/master/docs/nydus-design.md#2-rafs
Quoted 54 lines
Quoted 3 lines
showing you that EROFS doesn't currently support the functionality provided by PuzzleFS: the deduplication across multiple images.
No, EROFS supports external devices/blobs to keep a lot of chunks too (as dictionary to share data among images), but clearly it has the upper limit.
But PuzzleFS just treat each individual chunk as a seperate file, that will cause unavoidable "open arbitary number of files on reading, even in page fault context".
Quoted 26 lines
Quoted 9 lines
I believe they basically equal to your
Unified sizes, so the result isYour unified size EROFS (LZ4HC,12,64k) 54.2 PuzzleFS compressed 53? EROFS (DEFLATE,9,32k) 46.4Quoted 6 lines
That is why I used your 53 unified size to show EROFS is much smaller than PuzzleFS.
The reason why EROFS and SquashFS doesn't have the
Total Sizes is just because we cannot store every individual chunk into some seperate file.Well storing individual chunks into separate files is the entire point of PuzzleFS.
Quoted 7 lines
Currently, I have seen no reason to open arbitary kernel files (maybe hundreds due to large folio feature at once) in the page fault context. If I modified
mkfs.erofstool, I could give some similar numbers, but I don't want to waste time now due toopen arbitary kernel files in the page fault context.Quoted 8 lines
Quoted 5 lines
As I said, if PuzzleFS finally upstream some work to open kernel files in page fault context, I will definitely work out the same feature for EROFS soon, but currently I don't do that just because it's very controversal and no in-tree kernel filesystem does that.
The PuzzleFS kernel filesystem driver is still in an early POC stage, so there's still a lot more work to be done.
I suggest that you could just ask FS/MM folks about this ("open kernel files when reading in the page fault") first.
If they say "no", I suggest please don't waste on this anymore.
Thanks,
Gao Xiang
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
Quoted 1 line · Ariel Miculas
On 24/09/26 07:23, Gao Xiang wrote:
...
Quoted 6 lines
Quoted 6 lines
Quoted 3 lines
It might be a fair comparison, but that's not how container images are distributed. You're trying to argue that I should just use EROFS and I'm
First, OCI layer is just distributed like what I said.
Quoted 7 lines
Quoted 4 lines
For example, I could introduce some common blobs to keep
chunks as chunk dictionary. And then the each image
will be just some index, and all data will be deduplicated. That is also what Nydus works.I don't really follow what Nydus does. Here [1] it says they're using fixed size chunks of 1 MB. Where is the CDC step exactly?
Dragonfly Nydus uses fixed-size chunks of 1MiB by default with limited external blobs as chunk dictionaries. And ComposeFS uses per-file blobs.
Currently, Both are all EROFS users using different EROFS features. EROFS itself supports fixed-size chunks (unencoded), variable-sized extents (encoded, CDC optional) and limited external blobs.
Honestly, for your testload (10 versions of ubuntu:jammy), I don't think CDC made a significant difference in the final result compared to per-file blobs likewise. Because most of the files in these images are identical, I think there are only binary differences due to CVE fixes or similar issues. Maybe delta compression could do more help, but I never try this. So as I asked in [1], does ComposeFS already meet your requirement?
Again, EROFS could keep every different extent (or each chunk, whatever) as a seperate file with minor update, it's a trivial stuff for some userspace archive system, but IMO it's controversal for an in-tree kernel filesystem.
[1] https://github.com/project-machine/puzzlefs/issues/114#issuecomment-2369971291
Thanks,
Gao Xiang
Re: [RFC PATCH 03/24] erofs: add Errno in Rust
In reply to Ariel MiculasQuoted 24 lines · Ariel Miculas via Linux-erofs
Quoted 23 lines · Gao Xiang
Quoted 3 lines · Gao Xiang
On 2024/9/26 05:45, Ariel Miculas wrote:
...
Quoted 8 lines
Quoted 6 lines
I honestly don't see how it would look good if they're not using the existing filesystem abstractions. And I'm not convinced that Rust in the kernel would be useful in any way without the many subsystem abstractions which were implemented by the Rust for Linux team for the past few years.
So let's see the next version.
Some more words, regardless of in-tree "
fs/xfs/libxfs",
you also claimed "Another goal is to share the same code between user space and kernel space in order to provide one secure implementation." for example in [1].
Quoted 6 lines
Quoted 2 lines
I wonder Rust kernel VFS abstraction is forcely used in your userspace implementation, or (somewhat) your argument is still broken here.
Of course the implementations cannot be identical, but there is a lot of shared code between the user space and kernel space PuzzleFS implementations. The user space implementation uses the fuser [1] crate
If you know what you're doing, you may know what Yiyang is doing here, he will just form a Rust EROFS core logic and upstream later.
Thanks,
Gao Xiang
RFC PATCH 4/24 erofs: add xattrs data structure in Rust
In reply to Yiyang WuThis patch introduces on-disk and runtime data structure of Extended Attributes implementation in erofs_sys crate. This will be later used to implement the op handler.
--- fs/erofs/rust/erofs_sys.rs | 12 +++ fs/erofs/rust/erofs_sys/xattrs.rs | 124 ++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 fs/erofs/rust/erofs_sys/xattrs.rs
fs/erofs/rust/erofs_sys.rs +12 -0
@@ -25,4 +25,16 @@pub (crate )mod errnos;pub (crate )mod superblock; +pub (crate )mod xattrs;pub (crate )use errnos::Errno ; + +/// Helper macro to round up or down a number. +#[macro_export] +macro_rules! round { + (UP , $x: expr, $y: expr) => { + ($x + $y -1 ) / $y * $y + }; + (DOWN , $x: expr, $y: expr) => { + ($x / $y) * $y + }; +}
fs/erofs/rust/erofs_sys/xattrs.rs new file +124 -0
124 lines not on this page · raw message
Signature · 1 line
2.46.0
RFC PATCH 5/24 erofs: add inode data structure in Rust
In reply to Yiyang WuThis patch introduces the same on-disk erofs data structure in rust and also introduces multiple helpers for inode i_format and chunk_indexing and later can be used to implement map_blocks.
--- fs/erofs/rust/erofs_sys.rs | 1 + fs/erofs/rust/erofs_sys/inode.rs | 291 +++++++++++++++++++++++++++++++ 2 files changed, 292 insertions(+) create mode 100644 fs/erofs/rust/erofs_sys/inode.rs
fs/erofs/rust/erofs_sys.rs +1 -0
@@ -24,6 +24,7 @@pub (crate )type PosixResult <T > =Result <T ,Errno >;pub (crate )mod errnos; +pub (crate )mod inode;pub (crate )mod superblock;pub (crate )mod xattrs;pub (crate )use errnos::Errno ;
fs/erofs/rust/erofs_sys/inode.rs new file +291 -0
291 lines not on this page · raw message
Signature · 1 line
2.46.0
Re: [External Mail][RFC PATCH 05/24] erofs: add inode data structure in Rust
Quoted 186 lines · Yiyang Wu via Linux-erofs
This patch introduces the same on-disk erofs data structure in rust and also introduces multiple helpers for inode
i_formatandchunk_indexingand later can be used to implementmap_blocks.Signed-off-by Yiyang Wu <[email protected]>--- fs/erofs/rust/erofs_sys.rs | 1 + fs/erofs/rust/erofs_sys/inode.rs | 291 +++++++++++++++++++++++++++++++ 2 files changed, 292 insertions(+)create mode 100644
fs/erofs/rust/erofs_sys/inode.rsfs/erofs/rust/erofs_sys.rs +1
-0@@ -24,6 +24,7 @@pub (crate )type PosixResult <T > =Result <T ,Errno >;pub (crate )mod errnos; +pub (crate )mod inode;pub (crate )mod superblock;pub (crate )mod xattrs;pub (crate )use errnos::Errno ;fs/erofs/rust/erofs_sys/inode.rs new file +156
-0@@ -0,0 +1,291 @@ +use super ::xattrs::*; +use super ::*; +use core::ffi::*; +use core::mem::size_of; + +/// Represents the compact bitfield of the Erofs Inode format. +#[repr(transparent)] +#[derive( Clone , Copy )] +pub (crate )struct Format (u16 ); + +pub (crate )const INODE_VERSION_MASK :u16 =0x1 ; +pub (crate )const INODE_VERSION_BIT :u16 =0 ; + +pub (crate )const INODE_LAYOUT_BIT :u16 =1 ; +pub (crate )const INODE_LAYOUT_MASK :u16 =0x7 ; + +/// Helper macro to extract property from the bitfield. +macro_rules! extract { + ($name: expr, $bit: expr, $mask: expr) => { + ($name >> $bit) & ($mask) + }; +} + +/// The Version of the Inode which represents whether this inode is extended or compact. +/// Extended inodes have more infos about nlinks + mtime. +/// This is documented in https://erofs.docs.kernel.org/en/latest/core_ondisk.html#inodes +#[repr( C )] +#[derive( Clone , Copy )] +pub (crate )enum Version { +Compat , +Extended , +Unknown , +} + +/// Represents the data layout backed by the Inode. +/// As Documented in https://erofs.docs.kernel.org/en/latest/core_ondisk.html#inode-data-layouts +#[repr( C )] +#[derive( Clone , Copy , PartialEq )] +pub (crate )enum Layout { +FlatPlain , +CompressedFull , +FlatInline , +CompressedCompact , +Chunk , +Unknown , +} + +#[repr( C )] +#[allow(non_camel_case_types)] +#[derive( Clone , Copy , Debug , PartialEq )] +pub (crate )enum Type { +Regular , +Directory , +Link , +Character , +Block , +Fifo , +Socket , +Unknown , +} + +/// This is format extracted from i_format bit representation. +/// This includes various infos and specs about the inode. +impl Format { +pub (crate )fn version (&self) ->Version { +match extract! (self.0 ,INODE_VERSION_BIT ,INODE_VERSION_MASK ) { +0 =>Version ::Compat , +1 =>Version ::Extended , + _ =>Version ::Unknown , + } + } + +pub (crate )fn layout (&self) ->Layout { +match extract! (self.0 ,INODE_LAYOUT_BIT ,INODE_LAYOUT_MASK ) { +0 =>Layout ::FlatPlain , +1 =>Layout ::CompressedFull , +2 =>Layout ::FlatInline , +3 =>Layout ::CompressedCompact , +4 =>Layout ::Chunk , + _ =>Layout ::Unknown , + } + } +} + +/// Represents the compact inode which resides on-disk. +/// This is documented in https://erofs.docs.kernel.org/en/latest/core_ondisk.html#inodes +#[repr( C )] +#[derive( Clone , Copy )] +pub (crate )struct CompactInodeInfo { +pub (crate ) i_format:Format , +pub (crate ) i_xattr_icount:u16 , +pub (crate ) i_mode:u16 , +pub (crate ) i_nlink:u16 , +pub (crate ) i_size:u32 , +pub (crate ) i_reserved: [u8 ;4 ], +pub (crate ) i_u: [u8 ;4 ], +pub (crate ) i_ino:u32 , +pub (crate ) i_uid:u16 , +pub (crate ) i_gid:u16 , +pub (crate ) i_reserved2: [u8 ;4 ], +} + +/// Represents the extended inode which resides on-disk. +/// This is documented in https://erofs.docs.kernel.org/en/latest/core_ondisk.html#inodes +#[repr( C )] +#[derive( Clone , Copy )] +pub (crate )struct ExtendedInodeInfo { +pub (crate ) i_format:Format , +pub (crate ) i_xattr_icount:u16 , +pub (crate ) i_mode:u16 , +pub (crate ) i_reserved: [u8 ;2 ], +pub (crate ) i_size:u64 , +pub (crate ) i_u: [u8 ;4 ], +pub (crate ) i_ino:u32 , +pub (crate ) i_uid:u32 , +pub (crate ) i_gid:u32 , +pub (crate ) i_mtime:u64 , +pub (crate ) i_mtime_nsec:u32 , +pub (crate ) i_nlink:u32 , +pub (crate ) i_reserved2: [u8 ;16 ], +} + +/// Represents the inode info which is either compact or extended. +#[derive( Clone , Copy )] +pub (crate )enum InodeInfo { +Extended (ExtendedInodeInfo ), +Compact (CompactInodeInfo ), +} + +pub (crate )const CHUNK_BLKBITS_MASK :u16 =0x1f ; +pub (crate )const CHUNK_FORMAT_INDEX_BIT :u16 =0x20 ; + +/// Represents on-disk chunk index of the file backing inode. +#[repr( C )] +#[derive( Clone , Copy , Debug )] +pub (crate )struct ChunkIndex { +pub (crate ) advise:u16 , +pub (crate ) device_id:u16 , +pub (crate ) blkaddr:u32 , +} + +impl From <[u8 ;8 ]>for ChunkIndex { +fn from (u: [u8 ;8 ]) ->Self { +let advise = u16::from_le_bytes ([u[0 ], u[1 ]]); +let device_id = u16::from_le_bytes ([u[2 ], u[3 ]]); +let blkaddr = u32::from_le_bytes ([u[4 ], u[5 ], u[6 ], u[7 ]]); +ChunkIndex { + advise, + device_id, + blkaddr, + } + } +} + +/// Chunk format used for indicating the chunkbits and chunkindex. +#[repr( C )]
fs/erofs/rust/erofs_sys/inode.rs line 166
Quoted 10 lines
fs/erofs/rust/erofs_sys/inode.rs new file +10
-0+#[derive( Clone , Copy , Debug )] +pub (crate )struct ChunkFormat (pub (crate )u16 ); + +impl ChunkFormat { +pub (crate )fn is_chunkindex (&self) ->bool { + self.0 &CHUNK_FORMAT_INDEX_BIT !=0 + } +pub (crate )fn chunkbits (&self) ->u16 { + self.0 &CHUNK_BLKBITS_MASK + }
It is recommended to add blank lines between code blocks. This problem exists in many places in this patch set.
Quoted 56 lines
fs/erofs/rust/erofs_sys/inode.rs new file +56
-0+} + +/// Represents the inode spec which is either data or device. +#[derive( Clone , Copy , Debug )] +#[repr( u32 )] +pub (crate )enum Spec { +Chunk (ChunkFormat ), +RawBlk (u32 ), +Device (u32 ), +CompressedBlocks (u32 ), +Unknown , +} + +/// Convert the spec from the format of the inode based on the layout. +impl From <(&[u8 ;4 ],Layout )>for Spec { +fn from (value: (&[u8 ;4 ],Layout )) ->Self { +match value.1 { +Layout ::FlatInline |Layout ::FlatPlain =>Spec ::RawBlk (u32::from_le_bytes (*value.0 )), +Layout ::CompressedFull |Layout ::CompressedCompact => { +Spec ::CompressedBlocks (u32::from_le_bytes (*value.0 )) + } +Layout ::Chunk =>Self ::Chunk (ChunkFormat (u16::from_le_bytes ([value.0 [0 ], value.0 [1 ]]))), +// We don't support compressed inlines or compressed chunks currently. + _ =>Spec ::Unknown , + } + } +} + +/// Helper functions for Inode Info. +impl InodeInfo { +const S_IFMT :u16 =0o170000 ; +const S_IFSOCK :u16 =0o140000 ; +const S_IFLNK :u16 =0o120000 ; +const S_IFREG :u16 =0o100000 ; +const S_IFBLK :u16 =0o60000 ; +const S_IFDIR :u16 =0o40000 ; +const S_IFCHR :u16 =0o20000 ; +const S_IFIFO :u16 =0o10000 ; +const S_ISUID :u16 =0o4000 ; +const S_ISGID :u16 =0o2000 ; +const S_ISVTX :u16 =0o1000 ; +pub (crate )fn ino (&self) ->u32 { +match self { +Self ::Extended (extended) => extended.i_ino, +Self ::Compact (compact) => compact.i_ino, + } + } + +pub (crate )fn format (&self) ->Format { +match self { +Self ::Extended (extended) => extended.i_format, +Self ::Compact (compact) => compact.i_format, + } + } + +pub (crate )fn file_size (&self) ->Off {
fs/erofs/rust/erofs_sys/inode.rs line 232
Quoted 10 lines
fs/erofs/rust/erofs_sys/inode.rs new file +10
-0+match self { +Self ::Extended (extended) => extended.i_size, +Self ::Compact (compact) => compact.i_sizeas u64 , + } + } + +pub (crate )fn inode_size (&self) ->Off { +match self { +Self ::Extended (_) =>64 , +Self ::Compact (_) =>32 ,
Self::Extended(_) => size_of::<ExtendedInodeInfo>() as Off, Self::Compact(_) => size_of::<CompactInodeInfo>() as Off,
Quoted 6 lines
fs/erofs/rust/erofs_sys/inode.rs new file +6
-0+ } + } + +pub (crate )fn spec (&self) ->Spec { +let mode =match self { +Self ::Extended (extended) => extended.i_mode,
fs/erofs/rust/erofs_sys/inode.rs line 248
Quoted 10 lines
fs/erofs/rust/erofs_sys/inode.rs new file +10
-0+ Self::Compact(compact) => compact.i_mode, + }; + +let u =match self { +Self ::Extended (extended) => &extended.i_u, +Self ::Compact (compact) => &compact.i_u, + }; + +match mode &0o170000 { +0o40000 |0o100000 |0o120000 =>Spec ::from ((u, self.format ().layout ())),
match mode & Self::S_IFMT {
Self::S_IFDIR | Self::S_IFREG | Self::S_IFLNK => Spec::from((u, self.format().layout())),
Quoted 15 lines
fs/erofs/rust/erofs_sys/inode.rs new file +15
-0+// We don't support device inodes currently. + _ => Spec::Unknown, + } + } + +pub (crate )fn inode_type (&self) ->Type { +let mode =match self { +Self ::Extended (extended) => extended.i_mode, +Self ::Compact (compact) => compact.i_mode, + }; +match mode &Self ::S_IFMT { +Self ::S_IFDIR =>Type ::Directory ,// Directory +Self ::S_IFREG =>Type ::Regular ,// Regular File +Self ::S_IFLNK =>Type ::Link ,// Symbolic Link +Self ::S_IFIFO =>Type ::Fifo ,// FIFO
fs/erofs/rust/erofs_sys/inode.rs line 273
Quoted 10 lines
fs/erofs/rust/erofs_sys/inode.rs new file +10
-0+ Self::S_IFSOCK => Type::Socket, // Socket + Self::S_IFBLK => Type::Block, // Block + Self::S_IFCHR => Type::Character, // Character + _ => Type::Unknown, + } + } + +pub (crate )fn xattr_size (&self) ->Off { +match self { +Self ::Extended (extended) => {
if extended.i_xattr_icount == 0 {
return 0;
}to avoid subtract with overflow.
Thanks,
Jianan
Quoted 21 lines
fs/erofs/rust/erofs_sys/inode.rs new file +18
-1+size_of ::<XAttrSharedEntrySummary >()as Off + + (size_of ::<c_int >()as Off ) * (extended.i_xattr_icountas Off -1 ) + } + Self::Compact(_) => 0, + } + } + +pub (crate )fn xattr_count (&self) ->u16 { +match self { +Self ::Extended (extended) => extended.i_xattr_icount, +Self ::Compact (compact) => compact.i_xattr_icount, + } + } +} + +pub (crate )type CompactInodeInfoBuf = [u8 ;size_of ::<CompactInodeInfo >()]; +pub (crate )type ExtendedInodeInfoBuf = [u8 ;size_of ::<ExtendedInodeInfo >()]; +pub (crate )const DEFAULT_INODE_BUF :ExtendedInodeInfoBuf = [0 ;size_of ::<ExtendedInodeInfo >()]; --2.46.0
RFC PATCH 6/24 erofs: add alloc_helper in Rust
In reply to Yiyang WuIn normal rust, heap related operations are infallible meaning that they do not throw errors and Rust will panic in usermode instead. However in kernel, it will throw AllocError this module helps to bridge the gaps and returns Errno universally.
--- fs/erofs/rust/erofs_sys.rs | 1 + fs/erofs/rust/erofs_sys/alloc_helper.rs | 35 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 fs/erofs/rust/erofs_sys/alloc_helper.rs
fs/erofs/rust/erofs_sys.rs +1 -0
@@ -23,6 +23,7 @@/// to avoid naming conflicts. pub (crate )type PosixResult <T > =Result <T ,Errno >; +pub (crate )mod alloc_helper;pub (crate )mod errnos;pub (crate )mod inode;pub (crate )mod superblock;
fs/erofs/rust/erofs_sys/alloc_helper.rs new file +35 -0
@@ -0,0 +1,35 @@ +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later + +/// This module provides helper functions for the alloc crate +/// Note that in linux kernel, the allocation is fallible however in userland it is not. +/// Since most of the functions depend on infallible allocation, here we provide helper functions +/// so that most of codes don't need to be changed. + +#[cfg( CONFIG_EROFS_FS = "y" )] +use kernel::prelude::*; + +#[cfg(not( CONFIG_EROFS_FS = "y" ))] +use alloc::vec; + +use super ::*; +use alloc::boxed::Box ; +use alloc::vec::Vec ; + +pub (crate )fn push_vec <T >(v: &mut Vec <T >, value:T ) ->PosixResult <()> { + v.push (value,GFP_KERNEL ) + .map_or_else (|_|Err (Errno ::ENOMEM ), |_|Ok (())) +} + +pub (crate )fn extend_from_slice <T :Clone >(v: &mut Vec <T >, slice: &[T ]) ->PosixResult <()> { + v.extend_from_slice (slice,GFP_KERNEL ) + .map_or_else (|_|Err (Errno ::ENOMEM ), |_|Ok (())) +} + +pub (crate )fn heap_alloc <T >(value:T ) ->PosixResult <Box <T >> { +Box ::new (value,GFP_KERNEL ).map_or_else (|_|Err (Errno ::ENOMEM ), |v|Ok (v)) +} + +pub (crate )fn vec_with_capacity <T :Default +Clone >(capacity:usize ) ->PosixResult <Vec <T >> { +Vec ::with_capacity (capacity,GFP_KERNEL ).map_or_else (|_|Err (Errno ::ENOMEM ), |v|Ok (v)) +}
Signature · 1 line
2.46.0
RFC PATCH 7/24 erofs: add data abstraction in Rust
In reply to Yiyang WuIntroduce Buffer, Source, Backend traits.
Implement Uncompressed Backend and RefBuffer to be used in future data operations.
--- fs/erofs/rust/erofs_sys.rs | 1 + fs/erofs/rust/erofs_sys/data.rs | 62 +++++++++++++++++++ fs/erofs/rust/erofs_sys/data/backends.rs | 4 ++ .../erofs_sys/data/backends/uncompressed.rs | 39 ++++++++++++ 4 files changed, 106 insertions(+) create mode 100644 fs/erofs/rust/erofs_sys/data.rs create mode 100644 fs/erofs/rust/erofs_sys/data/backends.rs create mode 100644 fs/erofs/rust/erofs_sys/data/backends/uncompressed.rs
fs/erofs/rust/erofs_sys.rs +1 -0
@@ -24,6 +24,7 @@pub (crate )type PosixResult <T > =Result <T ,Errno >;pub (crate )mod alloc_helper; +pub (crate )mod data;pub (crate )mod errnos;pub (crate )mod inode;pub (crate )mod superblock;
fs/erofs/rust/erofs_sys/data.rs new file +62 -0
62 lines not on this page · raw message
fs/erofs/rust/erofs_sys/data/backends.rs new file +4 -0
@@ -0,0 +1,4 @@ +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later + +pub (crate )mod uncompressed;
fs/erofs/rust/erofs_sys/data/backends/uncompressed.rs new file +39 -0
@@ -0,0 +1,39 @@ +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later + +use super ::super ::*; + +pub (crate )struct UncompressedBackend <T > +where +T :Source , +{ + source:T , +} + +impl <T >Backend for UncompressedBackend <T > +where +T :Source , +{ +fn fill (&self, data: &mut [u8 ], offset:Off ) ->PosixResult <u64 > { + self.source.fill (data, offset) + } + +fn as_buf <'a>(&'a self, offset:Off , len:Off ) ->PosixResult <RefBuffer <'a>> { + self.source.as_buf (offset, len) + } +} + +impl <T :Source >UncompressedBackend <T > { +pub (crate )fn new (source:T ) ->Self { +Self { source } + } +} + +impl <T >From <T >for UncompressedBackend <T > +where +T :Source , +{ +fn from (value:T ) ->Self { +Self ::new (value) + } +}
Signature · 1 line
2.46.0
RFC PATCH 8/24 erofs: add device data structure in Rust
In reply to Yiyang WuThis patch introduce device data structure in Rust. It can later support chunk based block maps.
--- fs/erofs/rust/erofs_sys.rs | 1 + fs/erofs/rust/erofs_sys/devices.rs | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 fs/erofs/rust/erofs_sys/devices.rs
fs/erofs/rust/erofs_sys.rs +1 -0
@@ -25,6 +25,7 @@pub (crate )mod alloc_helper;pub (crate )mod data; +pub (crate )mod devices;pub (crate )mod errnos;pub (crate )mod inode;pub (crate )mod superblock;
fs/erofs/rust/erofs_sys/devices.rs new file +28 -0
@@ -0,0 +1,28 @@ +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later + +use alloc::vec::Vec ; + +/// Device specification. +#[derive( Copy , Clone , Debug )] +pub (crate )struct DeviceSpec { +pub (crate ) tags: [u8 ;64 ], +pub (crate ) blocks:u32 , +pub (crate ) mapped_blocks:u32 , +} + +/// Device slot. +#[derive( Copy , Clone , Debug )] +#[repr( C )] +pub (crate )struct DeviceSlot { + tags: [u8 ;64 ], + blocks:u32 , + mapped_blocks:u32 , + reserved: [u8 ;56 ], +} + +/// Device information. +pub (crate )struct DeviceInfo { +pub (crate ) mask:u16 , +pub (crate ) specs:Vec <DeviceSpec >, +}
Signature · 1 line
2.46.0
Re: [RFC PATCH 08/24] erofs: add device data structure in Rust
Yiyang Wu via Linux-erofs <[email protected]> 于2024年9月16日周一
Quoted 30 lines · 21:57写道:
This patch introduce device data structure in Rust. It can later support chunk based block maps.
Signed-off-by Yiyang Wu <[email protected]>--- fs/erofs/rust/erofs_sys.rs | 1 + fs/erofs/rust/erofs_sys/devices.rs | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 fs/erofs/rust/erofs_sys/devices.rsfs/erofs/rust/erofs_sys.rs +1
-0@@ -25,6 +25,7 @@pub (crate )mod alloc_helper;pub (crate )mod data; +pub (crate )mod devices;pub (crate )mod errnos;pub (crate )mod inode;pub (crate )mod superblock;fs/erofs/rust/erofs_sys/ renamed from fs/erofs/rust/erofs_sys/devices.rs +0
-0
devices.rs
new file mode 100644
index 000000000000..097676ee8720
fs/erofs/rust/erofs_sys/devices.rs line 9
Quoted 9 lines
fs/erofs/rust/erofs_sys/devices.rs new file +9
-0@@ -0,0 +1,28 @@ +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later + +use alloc::vec::Vec ; + +/// Device specification. +#[derive( Copy , Clone , Debug )] +pub (crate )struct DeviceSpec { +pub (crate ) tags: [u8 ;64 ],
I think we don't need to keep tags in the memory. It's not used in flatdev mode
or when mount with "-o device". It can be replaced with a string like the C version,
since it may represent a file path of variable length, or ignore this field for now.
Thanks,
Jianan
Quoted 23 lines
fs/erofs/rust/erofs_sys/devices.rs new file +19
-1+ pub(crate) blocks: u32, + pub(crate) mapped_blocks: u32, +} + +/// Device slot. +#[derive( Copy , Clone , Debug )] +#[repr( C )] +pub (crate )struct DeviceSlot { + tags: [u8 ;64 ], + blocks:u32 , + mapped_blocks:u32 , + reserved: [u8 ;56 ], +} + +/// Device information. +pub (crate )struct DeviceInfo { +pub (crate ) mask:u16 , +pub (crate ) specs:Vec <DeviceSpec >, +} --2.46.0
RFC PATCH 9/24 erofs: add continuous iterators in Rust
In reply to Yiyang WuThis patch adds a special iterator that is capable of iterating over a memory region in the granularity of a common page. This can be later used to read device buffer or fast symlink.
--- fs/erofs/rust/erofs_sys/data.rs | 2 + fs/erofs/rust/erofs_sys/data/raw_iters.rs | 6 ++ .../rust/erofs_sys/data/raw_iters/ref_iter.rs | 68 +++++++++++++++++++ .../rust/erofs_sys/data/raw_iters/traits.rs | 13 ++++ 4 files changed, 89 insertions(+) create mode 100644 fs/erofs/rust/erofs_sys/data/raw_iters.rs create mode 100644 fs/erofs/rust/erofs_sys/data/raw_iters/ref_iter.rs create mode 100644 fs/erofs/rust/erofs_sys/data/raw_iters/traits.rs
fs/erofs/rust/erofs_sys/data.rs +2 -0
@@ -1,6 +1,8 @@// Copyright 2024 Yiyang Wu // SPDX-License-Identifier: MIT or GPL-2.0-or-later pub (crate )mod backends; +pub (crate )mod raw_iters; +use super ::superblock::*;use super ::*;/// Represent some sort of generic data source. This cound be file, memory or even network.
fs/erofs/rust/erofs_sys/data/raw_iters.rs new file +6 -0
@@ -0,0 +1,6 @@ +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later + +pub (crate )mod ref_iter; +mod traits; +pub (crate )use traits::*;
fs/erofs/rust/erofs_sys/data/raw_iters/ref_iter.rs new file +68 -0
68 lines not on this page · raw message
fs/erofs/rust/erofs_sys/data/raw_iters/traits.rs new file +13 -0
@@ -0,0 +1,13 @@ +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later + +use super ::super ::*; + +/// Represents a basic iterator over a range of bytes from data backends. +/// Note that this is skippable and can be used to move the iterator's cursor forward. +pub (crate )trait ContinuousBufferIter <'a>: +Iterator <Item =PosixResult <RefBuffer <'a>>> +{ +fn advance_off (&mut self, offset:Off ); +fn eof (&self) ->bool ; +}
Signature · 1 line
2.46.0
RFC PATCH 10/24 erofs: add device_infos implementation in Rust
In reply to Yiyang WuAdd device_infos implementation in rust. It will later be used to be put inside the SuperblockInfo. This mask and spec can later be used to chunk-based image file block mapping.
--- fs/erofs/rust/erofs_sys/devices.rs | 47 ++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+)
fs/erofs/rust/erofs_sys/devices.rs +47 -0
@@ -1,6 +1,10 @@// Copyright 2024 Yiyang Wu // SPDX-License-Identifier: MIT or GPL-2.0-or-later +use super ::alloc_helper::*; +use super ::data::raw_iters::*; +use super ::data::*; +use super ::*;use alloc::vec::Vec ;/// Device specification.
@@ -21,8 +25,51 @@ pub(crate) struct DeviceSlot { reserved: [u8; 56], } +impl From <[u8 ;128 ]>for DeviceSlot { +fn from (data: [u8 ;128 ]) ->Self { +Self { + tags: data[0 ..64 ].try_into ().unwrap (), + blocks: u32::from_le_bytes ([data[64 ], data[65 ], data[66 ], data[67 ]]), + mapped_blocks: u32::from_le_bytes ([data[68 ], data[69 ], data[70 ], data[71 ]]), + reserved: data[72 ..128 ].try_into ().unwrap (), + } + } +} +/// Device information. pub (crate )struct DeviceInfo {pub (crate ) mask:u16 ,pub (crate ) specs:Vec <DeviceSpec >, } + +pub (crate )fn get_device_infos <'a>( + iter: &mut (dyn ContinuousBufferIter <'a> + 'a), +) ->PosixResult <DeviceInfo > { +let mut specs =Vec ::new (); +for datain iter { +let buffer = data?; +let mut cur:usize =0 ; +let len = buffer.content ().len (); +while cur +128 <= len { +let slot_data: [u8 ;128 ] = buffer.content ()[cur..cur +128 ].try_into ().unwrap (); +let slot =DeviceSlot ::from (slot_data); + cur +=128 ; +push_vec ( + &mut specs, +DeviceSpec { + tags: slot.tags, + blocks: slot.blocks, + mapped_blocks: slot.mapped_blocks, + }, + )?; + } + } + +let mask =if specs.is_empty () { +0 + }else { + (1 << (specs.len ().ilog2 () +1 )) -1 + }; + +Ok (DeviceInfo { mask, specs }) +}
Signature · 1 line
2.46.0
Re: [RFC PATCH 10/24] erofs: add device_infos implementation in Rust
Quoted 45 lines · Yiyang Wu via Linux-erofs <[email protected]> 于2024年9月16日周一 21:57写道:
Add
device_infosimplementation in rust. It will later be used to be put inside the SuperblockInfo. This mask and spec can later be used to chunk-based image file block mapping.Signed-off-by Yiyang Wu <[email protected]>--- fs/erofs/rust/erofs_sys/devices.rs | 47 ++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+)fs/erofs/rust/erofs_sys/devices.rs +15
-0@@ -1,6 +1,10 @@// Copyright 2024 Yiyang Wu // SPDX-License-Identifier: MIT or GPL-2.0-or-later +use super ::alloc_helper::*; +use super ::data::raw_iters::*; +use super ::data::*; +use super ::*;use alloc::vec::Vec ;/// Device specification. @@ -21,8 +25,51 @@ pub(crate) struct DeviceSlot { reserved: [u8; 56], } +impl From <[u8 ;128 ]>for DeviceSlot { +fn from (data: [u8 ;128 ]) ->Self { +Self { + tags: data[0 ..64 ].try_into ().unwrap (), + blocks: u32::from_le_bytes ([data[64 ], data[65 ], data[66 ], data[67 ]]), + mapped_blocks: u32::from_le_bytes ([data[68 ], data[69 ], data[70 ], data[71 ]]), + reserved: data[72 ..128 ].try_into ().unwrap (), + } + } +} +/// Device information. pub (crate )struct DeviceInfo {pub (crate ) mask:u16 ,pub (crate ) specs:Vec <DeviceSpec >, }
fs/erofs/rust/erofs_sys/devices.rs pub(crate) struct DeviceSlot { · line 53
Quoted 10 lines
fs/erofs/rust/erofs_sys/devices.rs +10
-0+ +pub (crate )fn get_device_infos <'a>( + iter: &mut (dyn ContinuousBufferIter <'a> + 'a), +) ->PosixResult <DeviceInfo > { +let mut specs =Vec ::new (); +for datain iter { +let buffer = data?; +let mut cur:usize =0 ; +let len = buffer.content ().len (); +while cur +128 <= len {
It is better to use macros instead of hardcode, like:
const EROFS_DEVT_SLOT_SIZE: usize = size_of::<DeviceSlot>(); Also works to the other similar usages in this patch set.
Thanks,
Jianan
Quoted 26 lines
fs/erofs/rust/erofs_sys/devices.rs +22
-1+let slot_data: [u8 ;128 ] = buffer.content ()[cur..cur +128 ].try_into ().unwrap (); +let slot =DeviceSlot ::from (slot_data); + cur +=128 ; +push_vec ( + &mut specs, +DeviceSpec { + tags: slot.tags, + blocks: slot.blocks, + mapped_blocks: slot.mapped_blocks, + }, + )?; + } + } + +let mask =if specs.is_empty () { +0 + }else { + (1 << (specs.len ().ilog2 () +1 )) -1 + }; + +Ok (DeviceInfo { mask, specs }) +} --2.46.0
RFC PATCH 11/24 erofs: add map data structure in Rust
In reply to Yiyang WuThis patch introduce core map flags and runtime map data structure in Rust. This will later be used to do iomapping.
--- fs/erofs/rust/erofs_sys.rs | 1 + fs/erofs/rust/erofs_sys/map.rs | 45 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 fs/erofs/rust/erofs_sys/map.rs
fs/erofs/rust/erofs_sys.rs +1 -0
@@ -28,6 +28,7 @@pub (crate )mod devices;pub (crate )mod errnos;pub (crate )mod inode; +pub (crate )mod map;pub (crate )mod superblock;pub (crate )mod xattrs;pub (crate )use errnos::Errno ;
fs/erofs/rust/erofs_sys/map.rs new file +45 -0
45 lines not on this page · raw message
Signature · 1 line
2.46.0
RFC PATCH 12/24 erofs: add directory entry data structure in Rust
In reply to Yiyang WuThis patch adds DirentDesc and DirCollection in Rust. It will later be used as helper to read_dir and lookup operations.
--- fs/erofs/rust/erofs_sys.rs | 1 + fs/erofs/rust/erofs_sys/dir.rs | 98 ++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 fs/erofs/rust/erofs_sys/dir.rs
fs/erofs/rust/erofs_sys.rs +1 -0
@@ -26,6 +26,7 @@pub (crate )mod alloc_helper;pub (crate )mod data;pub (crate )mod devices; +pub (crate )mod dir;pub (crate )mod errnos;pub (crate )mod inode;pub (crate )mod map;
fs/erofs/rust/erofs_sys/dir.rs new file +98 -0
98 lines not on this page · raw message
Signature · 1 line
2.46.0
RFC PATCH 13/24 erofs: add runtime filesystem and inode in Rust
In reply to Yiyang WuThis patch introduces Filesystem Trait and Inode trait in Rust. It also implements a memory backed filesystem in Rust which can be later hooks up the metabuf system in erofs.
This patch also comes with a InodeCollection trait which can be later be hooked up with the iget5_locked.
--- fs/erofs/rust/erofs_sys.rs | 2 +- fs/erofs/rust/erofs_sys/inode.rs | 106 ++++++++++++++++++++++ fs/erofs/rust/erofs_sys/superblock.rs | 42 ++++++++- fs/erofs/rust/erofs_sys/superblock/mem.rs | 61 +++++++++++++ 4 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 fs/erofs/rust/erofs_sys/superblock/mem.rs
fs/erofs/rust/erofs_sys.rs +1 -1
@@ -32,7 +32,7 @@pub (crate )mod map;pub (crate )mod superblock;pub (crate )mod xattrs; -pub (crate )use errnos::Errno ; +pub (crate )use errnos::{Errno ,Errno ::*};/// Helper macro to round up or down a number. #[macro_export]
fs/erofs/rust/erofs_sys/inode.rs +106 -0
112 lines not on this page · raw message
fs/erofs/rust/erofs_sys/superblock.rs +41 -1
53 lines not on this page · raw message
fs/erofs/rust/erofs_sys/superblock/mem.rs new file +61 -0
61 lines not on this page · raw message
Signature · 1 line
2.46.0
RFC PATCH 14/24 erofs: add block mapping capability in Rust
In reply to Yiyang WuImplement block mapping in rust and implement map iterators over Inode which will be used in data access.
--- fs/erofs/rust/erofs_sys/map.rs | 54 +++++++++++ fs/erofs/rust/erofs_sys/superblock.rs | 129 ++++++++++++++++++++++++++ 2 files changed, 183 insertions(+)
fs/erofs/rust/erofs_sys/map.rs +54 -0
@@ -1,7 +1,10 @@// Copyright 2024 Yiyang Wu // SPDX-License-Identifier: MIT or GPL-2.0-or-later +use super ::inode::*; +use super ::superblock::*;use super ::*; +pub (crate )const MAP_MAPPED :u32 =0x0001 ;pub (crate )const MAP_META :u32 =0x0002 ;pub (crate )const MAP_ENCODED :u32 =0x0004 ;
@@ -43,3 +46,54 @@ fn from(value: MapType) -> Self { }pub (crate )type MapResult =PosixResult <Map >; + +/// Iterates over the data map represented by an inode. +pub (crate )struct MapIter <'a, 'b,FS ,I > +where +FS :FileSystem <I >, +I :Inode , +{ + fs: &'aFS , + inode: &'bI , + offset:Off , + len:Off , +} + +impl <'a, 'b,FS ,I >MapIter <'a, 'b,FS ,I > +where +FS :FileSystem <I >, +I :Inode , +{ +pub (crate )fn new (fs: &'aFS , inode: &'bI , offset:Off ) ->Self { +Self { + fs, + inode, + offset, + len: inode.info ().file_size (), + } + } +} + +impl <'a, 'b,FS ,I >Iterator for MapIter <'a, 'b,FS ,I > +where +FS :FileSystem <I >, +I :Inode , +{ +type Item =MapResult ; +fn next (&mut self) ->Option <Self ::Item > { +if self.offset >= self.len { +None + }else { +let result = self.fs.map (self.inode, self.offset); +match result { +Ok (m) => { +let accessor = self.fs.superblock ().blk_access (m.physical.start); +let len = m.physical.len.min (accessor.len); + self.offset += len; +Some (Ok (m)) + } +Err (e) =>Some (Err (e)), + } + } + } +}
fs/erofs/rust/erofs_sys/superblock.rs +129 -0
149 lines not on this page · raw message
Signature · 1 line
2.46.0
RFC PATCH 15/24 erofs: add iter methods in filesystem in Rust
In reply to Yiyang WuImplement mapped iter that uses the MapIter and can yield data that is backed by EROFS inode.
Implement continuous_iter and mapped_iter for filesystem which can returns an iterator that yields raw data.
--- fs/erofs/rust/erofs_sys/data.rs | 2 + .../rust/erofs_sys/data/raw_iters/ref_iter.rs | 63 +++++++++++++++++++ .../rust/erofs_sys/data/raw_iters/traits.rs | 4 ++ fs/erofs/rust/erofs_sys/superblock.rs | 13 ++++ fs/erofs/rust/erofs_sys/superblock/mem.rs | 22 +++++++ 5 files changed, 104 insertions(+)
fs/erofs/rust/erofs_sys/data.rs +2 -0
@@ -2,6 +2,8 @@// SPDX-License-Identifier: MIT or GPL-2.0-or-later pub (crate )mod backends;pub (crate )mod raw_iters; +use super ::inode::*; +use super ::map::*;use super ::superblock::*;use super ::*;
fs/erofs/rust/erofs_sys/data/raw_iters/ref_iter.rs +63 -0
69 lines not on this page · raw message
fs/erofs/rust/erofs_sys/data/raw_iters/traits.rs +4 -0
@@ -3,6 +3,10 @@use super ::super ::*; +/// Represents a basic iterator over a range of bytes from data backends. +/// The access order is guided by the block maps from the filesystem. +pub (crate )trait BufferMapIter <'a>:Iterator <Item =PosixResult <RefBuffer <'a>>> {} +/// Represents a basic iterator over a range of bytes from data backends. /// Note that this is skippable and can be used to move the iterator's cursor forward. pub(crate) trait ContinuousBufferIter<'a>:
fs/erofs/rust/erofs_sys/superblock.rs +13 -0
@@ -5,6 +5,7 @@use alloc::boxed::Box ;use core::mem::size_of; +use super ::data::raw_iters::*;use super ::data::*;use super ::devices::*;use super ::inode::*;
@@ -274,6 +275,18 @@ fn map(&self, inode: &I, offset: Off) -> MapResult { _ => todo!(), } } + +fn mapped_iter <'b, 'a: 'b>( + &'a self, + inode: &'bI , + offset:Off , + ) ->PosixResult <Box <dyn BufferMapIter <'a> + 'b>>; + +fn continuous_iter <'a>( + &'a self, + offset:Off , + len:Off , + ) ->PosixResult <Box <dyn ContinuousBufferIter <'a> + 'a>>; }pub (crate )struct SuperblockInfo <I ,C ,T >
fs/erofs/rust/erofs_sys/superblock/mem.rs +22 -0
@@ -1,6 +1,7 @@// Copyright 2024 Yiyang Wu // SPDX-License-Identifier: MIT or GPL-2.0-or-later +use super ::alloc_helper::*;use super ::data::raw_iters::ref_iter::*;use super ::*;
@@ -33,6 +34,27 @@ fn as_filesystem(&self) -> &dyn FileSystem<I> { self } +fn mapped_iter <'b, 'a: 'b>( + &'a self, + inode: &'bI , + offset:Off , + ) ->PosixResult <Box <dyn BufferMapIter <'a> + 'b>> { +heap_alloc (RefMapIter ::new ( + &self.sb, + &self.backend, +MapIter ::new (self, inode, offset), + )) + .map (|v| vas Box <dyn BufferMapIter <'a> + 'b>) + } +fn continuous_iter <'a>( + &'a self, + offset:Off , + len:Off , + ) ->PosixResult <Box <dyn ContinuousBufferIter <'a> + 'a>> { +heap_alloc (ContinuousRefIter ::new (&self.sb, &self.backend, offset, len)) + .map (|v| vas Box <dyn ContinuousBufferIter <'a> + 'a>) + } +fn device_info (&self) -> &DeviceInfo { &self.device_info }
Signature · 1 line
2.46.0
RFC PATCH 16/24 erofs: implement dir and inode operations in Rust
In reply to Yiyang WuImplement dir ops and inode ops in Rust.
--- fs/erofs/rust/erofs_sys.rs | 1 + fs/erofs/rust/erofs_sys/data.rs | 4 ++ fs/erofs/rust/erofs_sys/operations.rs | 35 ++++++++++++++++ fs/erofs/rust/erofs_sys/superblock.rs | 59 +++++++++++++++++++++++++++ 4 files changed, 99 insertions(+) create mode 100644 fs/erofs/rust/erofs_sys/operations.rs
fs/erofs/rust/erofs_sys.rs +1 -0
@@ -30,6 +30,7 @@pub (crate )mod errnos;pub (crate )mod inode;pub (crate )mod map; +pub (crate )mod operations;pub (crate )mod superblock;pub (crate )mod xattrs;pub (crate )use errnos::{Errno ,Errno ::*};
fs/erofs/rust/erofs_sys/data.rs +4 -0
@@ -2,6 +2,7 @@// SPDX-License-Identifier: MIT or GPL-2.0-or-later pub (crate )mod backends;pub (crate )mod raw_iters; +use super ::dir::*;use super ::inode::*;use super ::map::*;use super ::superblock::*;
@@ -26,6 +27,9 @@ pub(crate) trait Backend {/// DirEntries. pub (crate )trait Buffer {fn content (&self) -> &[u8 ]; +fn iter_dir (&self) ->DirCollection <'_> { +DirCollection ::new (self.content ()) + } }/// Represents a buffer that holds a reference to a slice of data that
fs/erofs/rust/erofs_sys/operations.rs new file +35 -0
@@ -0,0 +1,35 @@ +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later + +use super ::inode::*; +use super ::superblock::*; +use super ::*; + +pub (crate )fn read_inode <'a,I ,C >( + filesystem: &'adyn FileSystem <I >, + collection: &'amut C , + nid:Nid , +) ->PosixResult <&'amut I > +where +I :Inode , +C :InodeCollection <I =I >, +{ + collection.iget (nid, filesystem) +} + +pub (crate )fn dir_lookup <'a,I ,C >( + filesystem: &'adyn FileSystem <I >, + collection: &'amut C , + inode: &I , + name: &str , +) ->PosixResult <&'amut I > +where +I :Inode , +C :InodeCollection <I =I >, +{ + filesystem + .find_nid (inode, name)? + .map_or (Err (Errno ::ENOENT ), |nid| { +read_inode (filesystem, collection, nid) + }) +}
fs/erofs/rust/erofs_sys/superblock.rs +59 -0
71 lines not on this page · raw message
Signature · 1 line
2.46.0
RFC PATCH 17/24 erofs: introduce Rust SBI to C
In reply to Yiyang WuThis patch introduces Rust opaque superblock info to C as s_fs_info. The original erofs_sb_info is embedded inside the rust opaque type and reexported by rewriting the original EROFS_SB/EROFS_I_SB macros.
This patch also provides a prototype of KernelInode, KernelInodeCollection so that the code can compile and it also implements the Metabuf Data Source by hooking up the original metabuf API defined in EROFS.
--- fs/erofs/Makefile | 2 +- fs/erofs/internal.h | 10 ++++++ fs/erofs/rust/kinode.rs | 49 ++++++++++++++++++++++++++ fs/erofs/rust/ksources.rs | 66 ++++++++++++++++++++++++++++++++++++ fs/erofs/rust/ksuperblock.rs | 30 ++++++++++++++++ fs/erofs/rust/mod.rs | 3 ++ fs/erofs/rust_bindings.h | 12 +++++++ fs/erofs/rust_helpers.c | 31 +++++++++++++++++ fs/erofs/rust_helpers.h | 21 ++++++++++++ fs/erofs/super.c | 15 ++++++-- fs/erofs/super_rs.rs | 50 +++++++++++++++++++++++++++ 11 files changed, 285 insertions(+), 4 deletions(-) create mode 100644 fs/erofs/rust/kinode.rs create mode 100644 fs/erofs/rust/ksources.rs create mode 100644 fs/erofs/rust/ksuperblock.rs create mode 100644 fs/erofs/rust_bindings.h create mode 100644 fs/erofs/rust_helpers.c create mode 100644 fs/erofs/rust_helpers.h
fs/erofs/Makefile +1 -1
@@ -9,4 +9,4 @@ erofs-$(CONFIG_EROFS_FS_ZIP_DEFLATE) += decompressor_deflate.o erofs-$(CONFIG_EROFS_FS_ZIP_ZSTD) += decompressor_zstd.o erofs-$(CONFIG_EROFS_FS_BACKED_BY_FILE) += fileio.o erofs-$(CONFIG_EROFS_FS_ONDEMAND) += fscache.o -erofs-$(CONFIG_EROFS_FS_RUST) += super_rs.o +erofs-$(CONFIG_EROFS_FS_RUST) += super_rs.o rust_helpers.o
fs/erofs/internal.h +10 -0
@@ -20,6 +20,10 @@#include <linux/iomap.h> #include "erofs_fs.h" +#ifdef CONFIG_EROFS_FS_RUST +#include "rust_bindings.h" +#endif +/* redefine pr_fmt "erofs: " */ #undef pr_fmt#define pr_fmt (fmt) "erofs: " fmt
@@ -178,8 +182,14 @@ struct erofs_sb_info {char *domain_id; }; +#ifdef CONFIG_EROFS_FS_RUST +#define EROFS_SB (sb) (*(struct erofs_sb_info **)(((void *)((sb)->s_fs_info)) + \ + EROFS_SB_INFO_OFFSET_RUST)) +#define EROFS_I_SB (inode) EROFS_SB((inode)->i_sb) +#else #define EROFS_SB (sb) ((struct erofs_sb_info *)(sb)->s_fs_info)#define EROFS_I_SB (inode) ((struct erofs_sb_info *)(inode)->i_sb->s_fs_info) +#endif /* Mount flags set via mount options or defaults */ #define EROFS_MOUNT_XATTR_USER 0x00000010
fs/erofs/rust/kinode.rs new file +49 -0
49 lines not on this page · raw message
fs/erofs/rust/ksources.rs new file +66 -0
66 lines not on this page · raw message
fs/erofs/rust/ksuperblock.rs new file +30 -0
@@ -0,0 +1,30 @@ +// Copyright 2024 Yiyang Wu +// SPDX-License-Identifier: MIT or GPL-2.0-or-later +use super ::erofs_sys::superblock::*; +use super ::kinode::*; +use alloc::boxed::Box ; +use core::{ffi::c_void, ptr::NonNull }; +use kernel::bindings::super_block; +use kernel::types::ForeignOwnable ; + +pub (crate )type KernelOpaque =NonNull <*mut c_void >; +/// KernelSuperblockInfo defined by embedded Kernel Inode +pub (crate )type KernelSuperblockInfo = +SuperblockInfo <KernelInode ,KernelInodeCollection ,KernelOpaque >; + +/// SAFETY: +/// Cast the c_void back to KernelSuperblockInfo. +/// This seems to be prune to some concurrency issues +/// but the fact is that only KernelInodeCollection field can have mutability. +/// However, it's backed by the original iget_locked5 and it's already preventing +/// any concurrency issues. So it's safe to be casted mutable here even if it's not backed by +/// Arc/Mutex instead of using generic method from Foreign Ownable which only provides +/// immutable reference casting which is not enough. +/// Since the pointer always live as long as this module exists, it's safe to declare it as static. +pub (crate )fn erofs_sbi (sb:NonNull <super_block >) -> &'staticmut KernelSuperblockInfo { +unsafe { &mut *(sb.as_ref ().s_fs_info).cast ::<KernelSuperblockInfo >() } +} + +pub (crate )fn free_sbi (sb:NonNull <super_block >) { +unsafe {Box ::<KernelSuperblockInfo >::from_foreign (sb.as_ref ().s_fs_info) }; +}
fs/erofs/rust/mod.rs +3 -0
@@ -2,3 +2,6 @@// SPDX-License-Identifier: MIT or GPL-2.0-or-later pub (crate )mod erofs_sys; +pub (crate )mod kinode; +pub (crate )mod ksources; +pub (crate )mod ksuperblock;
fs/erofs/rust_bindings.h new file +12 -0
@@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-later +// EROFS Rust Bindings Before VFS Patch Sets for Rust + +#ifndef __EROFS_RUST_BINDINGS_H +#define __EROFS_RUST_BINDINGS_H + +#include <linux/fs.h> + +extern const unsigned long EROFS_SB_INFO_OFFSET_RUST ; +extern void *erofs_alloc_sbi_rust (struct super_block *sb); +extern void *erofs_free_sbi_rust (struct super_block *sb); +#endif
fs/erofs/rust_helpers.c new file +31 -0
@@ -0,0 +1,31 @@ +#include "rust_helpers.h" + +static void erofs_init_metabuf_rust_helper (struct erofs_buf *buf, +struct super_block *sb, +struct erofs_sb_info *sbi) +{ +if (erofs_is_fileio_mode (sbi)) + buf->mapping =file_inode (sbi->fdev)->i_mapping; +else if (erofs_is_fscache_mode_rust_helper (sb, sbi)) + buf->mapping = sbi->s_fscache->inode->i_mapping; +else + buf->mapping = sb->s_bdev->bd_mapping; +} + +void *erofs_read_metabuf_rust_helper (struct super_block *sb, +struct erofs_sb_info *sbi, +erofs_off_t offset) +{ +struct erofs_buf buf = __EROFS_BUF_INITIALIZER; +erofs_init_metabuf_rust_helper (&buf, sb, sbi); +return erofs_bread (&buf, offset,EROFS_KMAP ); +} + +void erofs_put_metabuf_rust_helper (void *addr) +{ +erofs_put_metabuf (&(struct erofs_buf ){ + .base = addr, + .page =kmap_to_page (addr), + .kmap_type =EROFS_KMAP , + }); +}
fs/erofs/rust_helpers.h new file +21 -0
@@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-later +// This is a helpers collection to dodge the missing macros or inline functions in bindgen + +#ifndef __EROFS_RUST_HELPERS_H +#define __EROFS_RUST_HELPERS_H + +#include "internal.h" + +static inline bool erofs_is_fscache_mode_rust_helper (struct super_block *sb, +struct erofs_sb_info *sbi) +{ +return IS_ENABLED (CONFIG_EROFS_FS_ONDEMAND ) && + !erofs_is_fileio_mode (sbi) && !sb->s_bdev; +} + +void *erofs_read_metabuf_rust_helper (struct super_block *sb, +struct erofs_sb_info *sbi, +erofs_off_t offset); +void erofs_put_metabuf_rust_helper (void *addr); + +#endif
fs/erofs/super.c +12 -3
@@ -586,9 +586,12 @@ static void erofs_set_sysfs_name(struct super_block *sb)static int erofs_fc_fill_super (struct super_block *sb,struct fs_context *fc) {struct inode *inode; -struct erofs_sb_info *sbi =EROFS_SB (sb); +struct erofs_sb_info *sbi;int err; - +#ifdef CONFIG_EROFS_FS_RUST + sb->s_fs_info =erofs_alloc_sbi_rust (sb); +#endif + sbi =EROFS_SB (sb); sb->s_magic =EROFS_SUPER_MAGIC ; sb->s_flags |=SB_RDONLY |SB_NOATIME ; sb->s_maxbytes =MAX_LFS_FILESIZE ;
@@ -809,7 +812,13 @@ static int erofs_init_fs_context(struct fs_context *fc)static void erofs_kill_sb (struct super_block *sb) { -struct erofs_sb_info *sbi =EROFS_SB (sb); +struct erofs_sb_info *sbi; + +#ifdef CONFIG_EROFS_FS_RUST + sbi =erofs_free_sbi_rust (sb); +#else + sbi =EROFS_SB (sb); +#endif if ((IS_ENABLED (CONFIG_EROFS_FS_ONDEMAND ) && sbi->fsid) || sbi->fdev)kill_anon_super (sb);
fs/erofs/super_rs.rs +50 -0
53 lines not on this page · raw message
Signature · 1 line
2.46.0
RFC PATCH 18/24 erofs: introduce iget alternative to C
In reply to Yiyang WuThis patch introduces iget and fast symlink alternative written in Rust. After this patch, erofs_iget can be replaced with erofs_iget_rust.
Iget related test and set are lifted after this patch as rust_helpers.c will also use it to port the iget_locked to Rust.
--- fs/erofs/Makefile | 2 +- fs/erofs/inode.c | 8 ++++-- fs/erofs/inode_rs.rs | 59 ++++++++++++++++++++++++++++++++++++++++ fs/erofs/internal.h | 33 ++++++++++++++++++++++ fs/erofs/rust/kinode.rs | 29 +++++++++++++++++--- fs/erofs/rust_bindings.h | 12 ++++++++ fs/erofs/rust_helpers.c | 55 +++++++++++++++++++++++++++++++++++++ fs/erofs/rust_helpers.h | 4 ++- fs/erofs/super.c | 34 ++++++++++++++++++----- 9 files changed, 220 insertions(+), 16 deletions(-) create mode 100644 fs/erofs/inode_rs.rs
fs/erofs/Makefile +1 -1
@@ -9,4 +9,4 @@ erofs-$(CONFIG_EROFS_FS_ZIP_DEFLATE) += decompressor_deflate.o erofs-$(CONFIG_EROFS_FS_ZIP_ZSTD) += decompressor_zstd.o erofs-$(CONFIG_EROFS_FS_BACKED_BY_FILE) += fileio.o erofs-$(CONFIG_EROFS_FS_ONDEMAND) += fscache.o -erofs-$(CONFIG_EROFS_FS_RUST) += super_rs.o rust_helpers.o +erofs-$(CONFIG_EROFS_FS_RUST) += super_rs.o inode_rs.o rust_helpers.o
fs/erofs/inode.c +5 -3
@@ -269,7 +269,7 @@ int erofs_fill_inode(struct inode *inode)* ino_t is 32-bits on 32-bit arch. We have to squash the 64-bit value down * so that it will fit. */ -static ino_t erofs_squash_ino (erofs_nid_t nid) +ino_t erofs_squash_ino (erofs_nid_t nid) {ino_t ino = (ino_t )nid;
@@ -278,12 +278,12 @@ static ino_t erofs_squash_ino(erofs_nid_t nid)return ino; } -static int erofs_iget5_eq (struct inode *inode,void *opaque) +int erofs_iget5_eq (struct inode *inode,void *opaque) {return EROFS_I (inode)->nid == *(erofs_nid_t *)opaque; } -static int erofs_iget5_set (struct inode *inode,void *opaque) +int erofs_iget5_set (struct inode *inode,void *opaque) {const erofs_nid_t nid = *(erofs_nid_t *)opaque;
@@ -292,6 +292,7 @@ static int erofs_iget5_set(struct inode *inode, void *opaque)return 0 ; } +#ifndef CONFIG_EROFS_FS_RUST struct inode *erofs_iget (struct super_block *sb,erofs_nid_t nid) {struct inode *inode;
@@ -312,6 +313,7 @@ struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid) }return inode; } +#endif int erofs_getattr(struct mnt_idmap *idmap, const struct path *path, struct kstat *stat, u32 request_mask,
fs/erofs/inode_rs.rs new file +59 -0
59 lines not on this page · raw message
fs/erofs/internal.h +33 -0
@@ -306,10 +306,20 @@ struct erofs_inode {#endif /* CONFIG_EROFS_FS_ZIP */ };/* the corresponding vfs inode */ +#ifndef CONFIG_EROFS_FS_RUST struct inode vfs_inode; +#endif }; +#ifdef CONFIG_EROFS_FS_RUST +#define EROFS_I (ptr) (*(struct erofs_inode **)(((void *)(ptr)) + \ + EROFS_I_OFFSET_RUST)) +#define EROFS_I_VFS (ptr) ((struct inode *)(((void *)(ptr)) + EROFS_VFS_INODE_OFFSET_RUST)) +#define EROFS_I_RUST (ptr) ((void *)(ptr) - EROFS_VFS_INODE_OFFSET_RUST) +#else #define EROFS_I (ptr) container_of(ptr, struct erofs_inode, vfs_inode) +#define EROFS_I_VFS (ptr) (&((struct erofs_inode *)(ptr))->vfs_inode) +#endif static inline erofs_off_t erofs_iloc (struct inode *inode) {
@@ -427,10 +437,18 @@ void erofs_onlinefolio_init(struct folio *folio);void erofs_onlinefolio_split (struct folio *folio);void erofs_onlinefolio_end (struct folio *folio,int err);int erofs_fill_inode (struct inode *inode); +ino_t erofs_squash_ino (erofs_nid_t nid); +int erofs_iget5_eq (struct inode *inode,void *opaque); +int erofs_iget5_set (struct inode *inode,void *opaque); +#ifdef CONFIG_EROFS_FS_RUST +#define erofs_iget erofs_iget_rust +#else struct inode *erofs_iget (struct super_block *sb,erofs_nid_t nid); +#endif int erofs_getattr (struct mnt_idmap *idmap,const struct path *path,struct kstat *stat,u32 request_mask,unsigned int query_flags); +int erofs_namei (struct inode *dir,const struct qstr *name,erofs_nid_t *nid,unsigned int *d_type);
@@ -538,6 +556,21 @@ static inline struct bio *erofs_fscache_bio_alloc(struct erofs_map_dev *mdev) {static inline void erofs_fscache_submit_bio (struct bio *bio) {}#endif +#ifdef CONFIG_EROFS_FS_RUST +extern int erofs_init_rust (void ); +extern void erofs_destroy_rust (void ); +extern void erofs_init_inode_rust (struct inode *inode); +extern void erofs_free_inode_rust (struct inode *inode); +#else +static inline int erofs_init_rust (void ) +{ +return 0 ; +} +static inline void erofs_destroy_rust (void ) {} +static inline void erofs_init_inode_rust (struct inode *inode) {} +static inline void erofs_free_inode_rust (struct inode *inode) {} +#endif +#define EFSCORRUPTED EUCLEAN/* Filesystem is corrupted */ #endif /* __EROFS_INTERNAL_H */
fs/erofs/rust/kinode.rs +25 -4
@@ -1,16 +1,23 @@// Copyright 2024 Yiyang Wu // SPDX-License-Identifier: MIT or GPL-2.0-or-later -use core::ffi::c_void; +use core::ffi::*;use core::mem::MaybeUninit ;use core::ptr::NonNull ;use kernel::bindings::{inode, super_block}; +use kernel::container_of; +use super ::erofs_sys::errnos::*;use super ::erofs_sys::inode::*;use super ::erofs_sys::superblock::*;use super ::erofs_sys::*; +extern "C" { +#[link_name = "erofs_iget_locked_rust_helper" ] +fn iget_locked (sb:NonNull <c_void >, nid:Nid ) -> *mut c_void ; +} +#[repr( C )] pub (crate )struct KernelInode {pub (crate ) info:MaybeUninit <InodeInfo >,
@@ -21,7 +28,12 @@ pub(crate) struct KernelInode {impl Inode for KernelInode {fn new (_sb: &SuperBlock , _info:InodeInfo , _nid:Nid ) ->Self { -unimplemented! (); +Self { + info:MaybeUninit ::uninit (), + nid:MaybeUninit ::uninit (), + k_inode:MaybeUninit ::uninit (), + k_opaque:MaybeUninit ::uninit (), + } }fn nid (&self) ->Nid {unsafe { self.nid.assume_init () }
@@ -37,8 +49,17 @@ pub(crate) struct KernelInodeCollection {impl InodeCollection for KernelInodeCollection {type I =KernelInode ; -fn iget (&mut self, _nid:Nid , _f: &dyn FileSystem <Self ::I >) ->PosixResult <&mut Self ::I > { -unimplemented! (); +fn iget (&mut self, nid:Nid , _f: &dyn FileSystem <Self ::I >) ->PosixResult <&mut Self ::I > { +// SAFETY: iget_locked is safe to call here. +let k_inode =unsafe {iget_locked (self.sb.cast (), nid) }; +if is_value_err (k_inode.cast ()) { +return Err (Errno ::from (k_inodeas i32 )); + }else { +let erofs_inode: &mut KernelInode = +// SAFETY: iget_locked returns a valid pointer to a vfs inode and it's embedded in a KernelInode. +unsafe { &mut *(container_of! (k_inode,KernelInode , k_inode)as *mut KernelInode ) }; +return Ok (erofs_inode); + } } }
fs/erofs/rust_bindings.h +12 -0
@@ -6,7 +6,19 @@#include <linux/fs.h> + +typedef u64 erofs_nid_t ; +typedef u64 erofs_off_t ; +/* data type for filesystem-wide blocks number */ +typedef u32 erofs_blk_t ; +extern const unsigned long EROFS_SB_INFO_OFFSET_RUST ; +extern const unsigned int EROFS_INODE_SIZE_RUST ; +extern const unsigned long EROFS_VFS_INODE_OFFSET_RUST ; +extern const long EROFS_I_OFFSET_RUST ; +extern void *erofs_alloc_sbi_rust (struct super_block *sb);extern void *erofs_free_sbi_rust (struct super_block *sb); +extern int erofs_iget5_eq_rust (struct inode *inode,void *opaque); +extern struct inode *erofs_iget_rust (struct super_block *sb,erofs_nid_t nid);#endif
fs/erofs/rust_helpers.c +55 -0
63 lines not on this page · raw message
fs/erofs/rust_helpers.h +3 -1
@@ -17,5 +17,7 @@ void *erofs_read_metabuf_rust_helper(struct super_block *sb,struct erofs_sb_info *sbi, erofs_off_t offset);void erofs_put_metabuf_rust_helper (void *addr); - +extern int erofs_fill_inode_rust (struct inode *inode,erofs_nid_t nid); +struct inode *erofs_iget_locked_rust_helper (struct super_block *sb, +erofs_nid_t nid);#endif
fs/erofs/super.c +27 -7
@@ -81,22 +81,23 @@ static int erofs_superblock_csum_verify(struct super_block *sb, void *sbdata)static void erofs_inode_init_once (void *ptr) { -struct erofs_inode *vi = ptr; - -inode_init_once (&vi->vfs_inode); +inode_init_once (EROFS_I_VFS (ptr)); +erofs_init_inode_rust (EROFS_I_VFS (ptr)); }static struct inode *erofs_alloc_inode (struct super_block *sb) { -struct erofs_inode *vi = +void *ptr =alloc_inode_sb (sb, erofs_inode_cachep,GFP_KERNEL ); -if (!vi) +if (!ptr)return NULL ; +#ifndef CONFIG_EROFS_FS_RUST /* zero out everything except vfs_inode */ -memset (vi,0 , offsetof(struct erofs_inode , vfs_inode)); -return &vi->vfs_inode; +memset (ptr,0 , offsetof(struct erofs_inode , vfs_inode)); +#endif +return EROFS_I_VFS (ptr); }static void erofs_free_inode (struct inode *inode)
@@ -106,7 +107,12 @@ static void erofs_free_inode(struct inode *inode)if (inode->i_op == &erofs_fast_symlink_iops)kfree (inode->i_link);kfree (vi->xattr_shared_xattrs); +erofs_free_inode_rust (inode); +#ifdef CONFIG_EROFS_FS_RUST +kmem_cache_free (erofs_inode_cachep,EROFS_I_RUST (inode)); +#else kmem_cache_free (erofs_inode_cachep, vi); +#endif }/* read variable-sized metadata, offset will be aligned by 4-byte */
@@ -871,13 +877,25 @@ static int __init erofs_module_init(void)erofs_check_ondisk_layout_definitions (); +#ifndef CONFIG_EROFS_FS_RUST erofs_inode_cachep =kmem_cache_create ("erofs_inode" ,sizeof (struct erofs_inode ),0 ,SLAB_RECLAIM_ACCOUNT |SLAB_ACCOUNT , erofs_inode_init_once); +#else + erofs_inode_cachep =kmem_cache_create ("erofs_inode_rust" , +EROFS_INODE_SIZE_RUST ,0 , +SLAB_RECLAIM_ACCOUNT |SLAB_ACCOUNT , + erofs_inode_init_once); +#endif +if (!erofs_inode_cachep)return -ENOMEM ; + err =erofs_init_rust (); +if (err) + goto rust_err; + err =erofs_init_shrinker ();if (err) goto shrinker_err;
@@ -904,6 +922,8 @@ static int __init erofs_module_init(void)erofs_exit_shrinker (); shrinker_err:kmem_cache_destroy (erofs_inode_cachep); +rust_err: +erofs_destroy_rust ();return err; }
Signature · 1 line
2.46.0
RFC PATCH 19/24 erofs: introduce namei alternative to C
In reply to Yiyang WuThis patch introduces erofs_lookup_rust and erofs_get_parent_rust written in Rust as an alternative to the original namei.c.
--- fs/erofs/Makefile | 2 +- fs/erofs/internal.h | 2 ++ fs/erofs/namei.c | 2 ++ fs/erofs/namei_rs.rs | 56 ++++++++++++++++++++++++++++++++++++++++ fs/erofs/rust_bindings.h | 4 ++- fs/erofs/super.c | 2 ++ 6 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 fs/erofs/namei_rs.rs
fs/erofs/Makefile +1 -1
@@ -9,4 +9,4 @@ erofs-$(CONFIG_EROFS_FS_ZIP_DEFLATE) += decompressor_deflate.o erofs-$(CONFIG_EROFS_FS_ZIP_ZSTD) += decompressor_zstd.o erofs-$(CONFIG_EROFS_FS_BACKED_BY_FILE) += fileio.o erofs-$(CONFIG_EROFS_FS_ONDEMAND) += fscache.o -erofs-$(CONFIG_EROFS_FS_RUST) += super_rs.o inode_rs.o rust_helpers.o +erofs-$(CONFIG_EROFS_FS_RUST) += super_rs.o inode_rs.o namei_rs.o rust_helpers.o
fs/erofs/internal.h +2 -0
@@ -442,6 +442,8 @@ int erofs_iget5_eq(struct inode *inode, void *opaque);int erofs_iget5_set (struct inode *inode,void *opaque);#ifdef CONFIG_EROFS_FS_RUST #define erofs_iget erofs_iget_rust +#define erofs_get_parent erofs_get_parent_rust +#define erofs_lookup erofs_lookup_rust#else struct inode *erofs_iget (struct super_block *sb,erofs_nid_t nid);#endif
fs/erofs/namei.c +2 -0
@@ -7,6 +7,7 @@#include "xattr.h" #include <trace/events/erofs.h> +#ifndef CONFIG_EROFS_FS_RUST struct erofs_qstr { const unsigned char *name; const unsigned char *end;
@@ -214,6 +215,7 @@ static struct dentry *erofs_lookup(struct inode *dir, struct dentry *dentry, inode =erofs_iget (dir->i_sb, nid);return d_splice_alias (inode, dentry); } +#endif const struct inode_operations erofs_dir_iops = { .lookup = erofs_lookup,
fs/erofs/namei_rs.rs new file +56 -0
56 lines not on this page · raw message
fs/erofs/rust_bindings.h +3 -1
@@ -6,7 +6,6 @@#include <linux/fs.h> -typedef u64 erofs_nid_t ;typedef u64 erofs_off_t ;/* data type for filesystem-wide blocks number */
@@ -21,4 +20,7 @@ extern void *erofs_alloc_sbi_rust(struct super_block *sb);extern void *erofs_free_sbi_rust (struct super_block *sb);extern int erofs_iget5_eq_rust (struct inode *inode,void *opaque);extern struct inode *erofs_iget_rust (struct super_block *sb,erofs_nid_t nid); +extern struct dentry *erofs_lookup_rust (struct inode *inode,struct dentry *dentry, +unsigned int flags); +extern struct dentry *erofs_get_parent_rust (struct dentry *dentry);#endif
fs/erofs/super.c +2 -0
@@ -554,6 +554,7 @@ static struct dentry *erofs_fh_to_parent(struct super_block *sb, erofs_nfs_get_inode); } +#ifndef CONFIG_EROFS_FS_RUST static struct dentry *erofs_get_parent (struct dentry *child) {erofs_nid_t nid;
@@ -565,6 +566,7 @@ static struct dentry *erofs_get_parent(struct dentry *child)return ERR_PTR (err);return d_obtain_alias (erofs_iget (child->d_sb, nid)); } +#endif static const struct export_operations erofs_export_ops = { .encode_fh = generic_encode_ino32_fh,
Signature · 1 line
2.46.0
Al Viro
maintainerRe: [RFC PATCH 19/24] erofs: introduce namei alternative to C
fs/erofs/namei_rs.rs line 26
Quoted 9 lines · Yiyang Wu
fs/erofs/namei_rs.rs new file +9
-0+/// Lookup function for dentry-inode lookup replacement. +#[no_mangle] +pub unsafe extern "C" fn erofs_lookup_rust ( + k_inode:NonNull <inode >, + dentry:NonNull <dentry >, + _flags:c_uint , +) -> *mut c_void { +// SAFETY: We are sure that the inode is a Kernel Inode since alloc_inode is called +let erofs_inode =unsafe { &*container_of! (k_inode.as_ptr (),KernelInode , k_inode) };
Ummm... A wrapper would be highly useful. And the reason why
it's safe is different - your function is called only via ->i_op->lookup, the is only one instance of inode_operations that has that ->lookup method, and the only place where an inode gets ->i_op set to that is erofs_fill_inode(). Which is always passed erofs_inode::vfs_inode.
fs/erofs/namei_rs.rs line 28
Quoted 2 lines
fs/erofs/namei_rs.rs new file +2
-0+// SAFETY: The super_block is initialized when the erofs_alloc_sbi_rust is called. +let sbi =erofs_sbi (unsafe {NonNull ::new (k_inode.as_ref ().i_sb).unwrap () });
Again, that calls for a wrapper - this time not erofs-specific;
inode->i_sb is *always* non-NULL, is assign-once and always points to live struct super_block instance at least until the call of destroy_inode().
Quoted 1 line
+ // SAFETY: this is backed by qstr which is c representation of a valid slice.
What is that sentence supposed to mean? Nevermind "why is it correct"...
fs/erofs/namei_rs.rs line 33
Quoted 4 lines
fs/erofs/namei_rs.rs new file +4
-0+ let name = unsafe { + core::str::from_utf8_unchecked(core::slice::from_raw_parts( + dentry.as_ref().d_name.name, + dentry.as_ref().d_name.__bindgen_anon_1.__bindgen_anon_1.len as usize,
Is that supposed to be an example of idiomatic Rust? I'm not
trying to be snide, but my interest here is mostly about safety of
access to VFS data structures. And ->d_name is _very_ unpleasant in
that respect; the locking rules required for its stability are subtle and hard to verify on manual code audit.
Current erofs_lookup() (and your version as well) *is* indeed
safe in that respect, but the proof (from filesystem POV) is that "it's called only as ->lookup() instance, so dentry is initially unhashed negative and will remain such until it's passed to d_splice_alias(); until that point it is guaranteed to have ->d_name and ->d_parent stable".
Note that once you _have_ called d_splice_alias(), you can't
count upon the ->d_name stability - or, indeed, upon ->d_name.name you've sampled still pointing to allocated memory.
For directory-modifying methods it's "stable, since parent is held
exclusive". Some internal function called from different environments? Well... Swear, look through the call graph and see what can be proven for each.
Expressing that kind of fun in any kind of annotations (Rust type
system included) is not pleasant. _Probably_ might be handled by a type that would be a dentry pointer with annotation along the lines "->d_name and ->d_parent of that one are stable". Then e.g. ->lookup() would take that thing as an argument and d_splice_alias() would consume it. ->mkdir() would get the same thing, etc. I hadn't tried to get that all way through (the amount of annotation churn in existing filesystems would be high and hard to split into reviewable patch series), so there might be dragons - and there definitely are places where the stability is proven in different ways (e.g. if dentry->d_lock is held, we have the damn thing stable; then there's a "take a safe snapshot of name" API; etc.).
I want to reduce the PITA of regular code audits. If, at
some point, Rust use in parts of the tree reduces that - wonderful. But then we'd better make sure that Rust-side uses _are_ safe, accurately annotated and easy to grep for. Because we'll almost certainly need to change method calling conventions at some points through all of that. Even if it's just the annotation-level, any such contract change (it is doable and quite a few had been done) will require going through the instances and checking how much massage will be needed in those. Opaque chunks like the above promise to make that very painful...
Re: [RFC PATCH 19/24] erofs: introduce namei alternative to C
Quoted 17 lines · Al Viro
fs/erofs/namei_rs.rs line 26
Quoted 9 lines · Yiyang Wu
fs/erofs/namei_rs.rs new file +9
-0+/// Lookup function for dentry-inode lookup replacement. +#[no_mangle] +pub unsafe extern "C" fn erofs_lookup_rust ( + k_inode:NonNull <inode >, + dentry:NonNull <dentry >, + _flags:c_uint , +) -> *mut c_void { +// SAFETY: We are sure that the inode is a Kernel Inode since alloc_inode is called +let erofs_inode =unsafe { &*container_of! (k_inode.as_ptr (),KernelInode , k_inode) };Ummm... A wrapper would be highly useful. And the reason why
it's safe is different - your function is called only via ->i_op->lookup, the is only one instance ofinode_operationsthat has that ->lookup method, and the only place where an inode gets ->i_op set to that iserofs_fill_inode(). Which is always passederofs_inode::vfs_inode.
So my original intention behind this is that all vfs_inodes come from that erofs_iget function and it's always gets initialized in this case And this just followes the same convention here. I can document this more precisely.
Quoted 8 lines
fs/erofs/namei_rs.rs line 28
Quoted 2 lines
fs/erofs/namei_rs.rs new file +2
-0+// SAFETY: The super_block is initialized when the erofs_alloc_sbi_rust is called. +let sbi =erofs_sbi (unsafe {NonNull ::new (k_inode.as_ref ().i_sb).unwrap () });Again, that calls for a wrapper - this time not erofs-specific;
inode->i_sbis *always* non-NULL, is assign-once and always points to live structsuper_blockinstance at least until the call ofdestroy_inode().
Will be modified correctly, I'm not a native speaker and I just can't find a better way, I will take my note here.
Quoted 9 lines
Quoted 1 line
+ // SAFETY: this is backed by qstr which is c representation of a valid slice.
What is that sentence supposed to mean? Nevermind "why is it correct"...
fs/erofs/namei_rs.rs line 33
Quoted 4 lines
fs/erofs/namei_rs.rs new file +4
-0+ let name = unsafe { + core::str::from_utf8_unchecked(core::slice::from_raw_parts( + dentry.as_ref().d_name.name, + dentry.as_ref().d_name.__bindgen_anon_1.__bindgen_anon_1.len as usize,
Quoted 6 lines
Is that supposed to be an example of idiomatic Rust? I'm not
trying to be snide, but my interest here is mostly about safety of
access to VFS data structures. And ->d_name is _very_ unpleasant in
that respect; the locking rules required for its stability are subtle and hard to verify on manual code audit.
Yeah, this code is pretty messed up. I just cannot find a better way to use this qstr in dentry. So the original C qstr is this.
```c
struct qstr {
union {
struct {
HASH_LEN_DECLARE;
};
u64 hash_len;
};
const unsigned char *name;
};
```The original C code can use this pretty easily.
```C
int erofs_namei(struct inode *dir, const struct qstr *name, erofs_nid_t *nid,
unsigned int *d_type)
{
int ndirents;
struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
struct erofs_dirent *de;
struct erofs_qstr qn;
if (!dir->i_size)
return -ENOENT;
qn.name = name->name;
qn.end = name->name + name->len;
buf.mapping = dir->i_mapping;
```But after bindgen, since Rust does not support any kinds of anonymous unions here it just converts to this.
```rust
#[repr(C)]
#[derive(Copy, Clone)]
pub struct qstr {
pub __bindgen_anon_1: qstr__bindgen_ty_1,
pub name: *const core::ffi::c_uchar,
}
#[repr(C)]
#[derive(Copy, Clone)]
pub union qstr__bindgen_ty_1 {
pub __bindgen_anon_1: qstr__bindgen_ty_1__bindgen_ty_1,
pub hash_len: u64_,
}
```And it just somehow degrades to this pure mess. :( I know this is stupid :(
Quoted 6 lines
Current
erofs_lookup()(and your version as well) *is* indeed
safe in that respect, but the proof (from filesystem POV) is that "it's called only as ->lookup() instance, so dentry is initially unhashed negative and will remain such until it's passed tod_splice_alias(); until that point it is guaranteed to have ->d_name and ->d_parent stable".
Quoted 8 lines
Note that once you _have_ called
d_splice_alias(), you can't
count upon the ->d_name stability - or, indeed, upon ->d_name.name you've sampled still pointing to allocated memory.For directory-modifying methods it's "stable, since parent is held
exclusive". Some internal function called from different environments? Well... Swear, look through the call graph and see what can be proven for each.
Sorry for my ignorance.
I mean i just borrowed the code from the fs/erofs/namei.c and i directly translated that into Rust code. That might be a problem that also exists in original working C code.
Quoted 11 lines
Expressing that kind of fun in any kind of annotations (Rust type
system included) is not pleasant. _Probably_ might be handled by a type that would be a dentry pointer with annotation along the lines "->d_name and ->d_parent of that one are stable". Then e.g. ->lookup() would take that thing as an argument andd_splice_alias()would consume it. ->mkdir() would get the same thing, etc. I hadn't tried to get that all way through (the amount of annotation churn in existing filesystems would be high and hard to split into reviewable patch series), so there might be dragons - and there definitely are places where the stability is proven in different ways (e.g. ifdentry->d_lockis held, we have the damn thing stable; then there's a "take a safe snapshot of name" API; etc.).
That's kinda interesting, I originally thought that VFS will make sure its d_name / d_parent is stable in the first place.
Again, I just don't have a full picture or understanding of VFS and my code is just basic translation of original C code, Maybe we can address this later.
Quoted 4 lines
I want to reduce the PITA of regular code audits. If, at
some point, Rust use in parts of the tree reduces that - wonderful. But then we'd better make sure that Rust-side uses _are_ safe, accurately annotated and easy to grep for.
Yeah, even a small portion of VFS gets abstracted can get things work smoothly, like inode and dentry data structure. From my understanding Rust can make some of stability issues you have mentioned compilation errors if it's correctly annotated.
Quoted 6 lines
Because we'll almost certainly need to
change method calling conventions at some points through all of that. Even if it's just the annotation-level, any such contract change (it is doable and quite a few had been done) will require going through the instances and checking how much massage will be needed in those. Opaque chunks like the above promise to make that very painful...
Certainly needs a lot of energy and efforts.
WE just need a guy who has full grasp of a lot of filesystems and Rust to solve this stuff altogether to avoid the abstraction degraded into some toy concepts.
Re: [RFC PATCH 19/24] erofs: introduce namei alternative to C
Quoted 18 lines · Yiyang Wu
Quoted 17 lines · Al Viro
fs/erofs/namei_rs.rs line 26
Quoted 9 lines · Yiyang Wu
fs/erofs/namei_rs.rs new file +9
-0+/// Lookup function for dentry-inode lookup replacement. +#[no_mangle] +pub unsafe extern "C" fn erofs_lookup_rust ( + k_inode:NonNull <inode >, + dentry:NonNull <dentry >, + _flags:c_uint , +) -> *mut c_void { +// SAFETY: We are sure that the inode is a Kernel Inode since alloc_inode is called +let erofs_inode =unsafe { &*container_of! (k_inode.as_ptr (),KernelInode , k_inode) };Ummm... A wrapper would be highly useful. And the reason why
it's safe is different - your function is called only via ->i_op->lookup, the is only one instance ofinode_operationsthat has that ->lookup method, and the only place where an inode gets ->i_op set to that iserofs_fill_inode(). Which is always passederofs_inode::vfs_inode.
Quoted 4 lines
So my original intention behind this is that all
vfs_inodescome from thaterofs_igetfunction and it's always gets initialized in this case And this just followes the same convention here. I can document this more precisely.
I think Al just would like a wrapper here, like the current C EROFS_I().
Quoted 11 lines
Quoted 8 lines
fs/erofs/namei_rs.rs line 28
Quoted 2 lines
fs/erofs/namei_rs.rs new file +2
-0+// SAFETY: The super_block is initialized when the erofs_alloc_sbi_rust is called. +let sbi =erofs_sbi (unsafe {NonNull ::new (k_inode.as_ref ().i_sb).unwrap () });Again, that calls for a wrapper - this time not erofs-specific;
inode->i_sbis *always* non-NULL, is assign-once and always points to live structsuper_blockinstance at least until the call ofdestroy_inode().Will be modified correctly, I'm not a native speaker and I just can't find a better way, I will take my note here.
Same here, like the current EROFS_I_SB().
Quoted 9 lines
Quoted 9 lines
Quoted 1 line
+ // SAFETY: this is backed by qstr which is c representation of a valid slice.
What is that sentence supposed to mean? Nevermind "why is it correct"...
Quoted 4 lines
fs/erofs/namei_rs.rs new file +4
-0+ let name = unsafe { + core::str::from_utf8_unchecked(core::slice::from_raw_parts( + dentry.as_ref().d_name.name, + dentry.as_ref().d_name.__bindgen_anon_1.__bindgen_anon_1.len as usize,
...
Quoted 6 lines
Quoted 6 lines
Current
erofs_lookup()(and your version as well) *is* indeed
safe in that respect, but the proof (from filesystem POV) is that "it's called only as ->lookup() instance, so dentry is initially unhashed negative and will remain such until it's passed tod_splice_alias(); until that point it is guaranteed to have ->d_name and ->d_parent stable".
Agreed.
Quoted 10 lines
Quoted 9 lines
Note that once you _have_ called
d_splice_alias(), you can't
count upon the ->d_name stability - or, indeed, upon ->d_name.name you've sampled still pointing to allocated memory.For directory-modifying methods it's "stable, since parent is held
exclusive". Some internal function called from different environments? Well... Swear, look through the call graph and see what can be proven for each.
Quoted 4 lines
Sorry for my ignorance.
I mean i just borrowed the code from thefs/erofs/namei.cand i directly translated that into Rust code. That might be a problem that also exists in original working C code.
As for EROFS (an immutable fs), I think after d_splice_alias(), d_name is still stable (since we don't have rename semantics likewise for now).
But as the generic filesystem POV, d_name access is actually tricky under RCU walk path indeed.
Quoted 13 lines
Quoted 12 lines
Expressing that kind of fun in any kind of annotations (Rust type
system included) is not pleasant. _Probably_ might be handled by a type that would be a dentry pointer with annotation along the lines "->d_name and ->d_parent of that one are stable". Then e.g. ->lookup() would take that thing as an argument andd_splice_alias()would consume it. ->mkdir() would get the same thing, etc. I hadn't tried to get that all way through (the amount of annotation churn in existing filesystems would be high and hard to split into reviewable patch series), so there might be dragons - and there definitely are places where the stability is proven in different ways (e.g. ifdentry->d_lockis held, we have the damn thing stable; then there's a "take a safe snapshot of name" API; etc.).
Quoted 5 lines
That's kinda interesting, I originally thought that VFS will make sure its
d_name/d_parentis stable in the first place.
Again, I just don't have a full picture or understanding of VFS and my code is just basic translation of original C code, Maybe we can address this later.
d_alloc will allocate an unhashed dentry which is almost unrecognized by VFS dcache (d_name is stable of course).
After d_splice_alias() and d_add(), rename() could change d_name. So either we take d_lock or with rcu_read_lock() to take a snapshot of d_name in the RCU walk path. That is my overall understanding.
But for EROFS, since we don't have rename, so it doesn't matter.
Thanks,
Gao Xiang
Al Viro
maintainerRe: [RFC PATCH 19/24] erofs: introduce namei alternative to C
Quoted 7 lines · Gao Xiang
Quoted 4 lines
Sorry for my ignorance.
I mean i just borrowed the code from thefs/erofs/namei.cand i directly translated that into Rust code. That might be a problem that also exists in original working C code.As for EROFS (an immutable fs), I think after
d_splice_alias(),d_nameis still stable (since we don't have rename semantics likewise for now).
Even on corrupted images? If you have two directories with entries that act as hardlinks to the same subdirectory, and keep hitting them on lookups, it will have to transplant the subtree between the parents.
Quoted 2 lines
But as the generic filesystem POV,
d_nameaccess is actually tricky under RCU walk path indeed.
->lookup() is never called in RCU mode.
Quoted 6 lines
Quoted 5 lines
That's kinda interesting, I originally thought that VFS will make sure its
d_name/d_parentis stable in the first place.
Again, I just don't have a full picture or understanding of VFS and my code is just basic translation of original C code, Maybe we can address this later.
Quoted 6 lines
d_allocwill allocate an unhashed dentry which is almost unrecognized by VFS dcache (d_nameis stable of course).After
d_splice_alias()andd_add(),rename()could changed_name. So either we taked_lockor withrcu_read_lock()to take a snapshot ofd_namein the RCU walk path. That is my overall understanding.
No, it's more complicated than that, sadly. ->d_name and ->d_parent are the trickiest parts of dentry field stability.
Quoted 1 line
But for EROFS, since we don't have rename, so it doesn't matter.
See above. IF we could guarantee that all filesystem images are valid and will remain so, life would be much simpler.
Al Viro
maintainerRe: [RFC PATCH 19/24] erofs: introduce namei alternative to C
Quoted 11 lines · Al Viro
Quoted 3 lines
After
d_splice_alias()andd_add(),rename()could changed_name. So either we taked_lockor withrcu_read_lock()to take a snapshot ofd_namein the RCU walk path. That is my overall understanding.No, it's more complicated than that, sadly. ->d_name and ->d_parent are the trickiest parts of dentry field stability.
Quoted 1 line
But for EROFS, since we don't have rename, so it doesn't matter.
See above. IF we could guarantee that all filesystem images are valid and will remain so, life would be much simpler.
In any case, currently it is safe - d_splice_alias() is the last thing done by erofs_lookup(). Just don't assume that names can't change in there - and the fewer places in filesystem touch ->d_name, the better.
In practice, for ->lookup() you are safe until after d_splice_alias() and for directory-modifying operations you are safe unless you start playing insane games with unlocking and relocking the parent directories (apparmorfs does; the locking is really obnoxious there). That covers the majority of ->d_name and ->d_parent accesses in filesystem code.
->d_hash() and ->d_compare() are separate story; I've posted a text on that last year (or this winter - not sure, will check once I get some sleep).
d_path() et.al. are taking care to do the right thing; those (and %pd format) can be used safely.
Anyway, I'm half-asleep at the moment and I'd rather leave writing these rules up until tomorrow. Sorry...
Re: [RFC PATCH 19/24] erofs: introduce namei alternative to C
Quoted 28 lines · Al Viro
Quoted 11 lines · Al Viro
Quoted 3 lines
After
d_splice_alias()andd_add(),rename()could changed_name. So either we taked_lockor withrcu_read_lock()to take a snapshot ofd_namein the RCU walk path. That is my overall understanding.No, it's more complicated than that, sadly. ->d_name and ->d_parent are the trickiest parts of dentry field stability.
Quoted 1 line
But for EROFS, since we don't have rename, so it doesn't matter.
See above. IF we could guarantee that all filesystem images are valid and will remain so, life would be much simpler.
In any case, currently it is safe -
d_splice_alias()is the last thing done byerofs_lookup(). Just don't assume that names can't change in there - and the fewer places in filesystem touch ->d_name, the better.In practice, for ->lookup() you are safe until after
d_splice_alias()and for directory-modifying operations you are safe unless you start playing insane games with unlocking and relocking the parent directories (apparmorfs does; the locking is really obnoxious there). That covers the majority of ->d_name and ->d_parent accesses in filesystem code.->d_hash() and ->d_compare() are separate story; I've posted a text on that last year (or this winter - not sure, will check once I get some sleep).
Quoted 5 lines
d_path()et.al. are taking care to do the right thing; those (and %pd format) can be used safely.Anyway, I'm half-asleep at the moment and I'd rather leave writing these rules up until tomorrow. Sorry...
Agreed, thanks for writing so many words on this!
Thanks,
Gao Xiang
Al Viro
maintainerRe: [RFC PATCH 19/24] erofs: introduce namei alternative to C
In reply to Al ViroQuoted 2 lines · Al Viro
Anyway, I'm half-asleep at the moment and I'd rather leave writing these rules up until tomorrow. Sorry...
[Below are the bits of my notes related to d_name and d_parent, with most of the unprintable parts thrown out and minimal markup added. Probably not all relevant notes are here - this has been culled from a bunch of files sitting around and I might've missed some]
NOTE: ->d_parent and ->d_name are by far the worst parts of dentry wrt stability rules. Code audits are not pleasant, to put it mildly. This covers the bits outside of VFS proper - verifying the locking rules is a separate story.
A Really Blunt Tool You Should Not Use.
======================================
All changes of ->d_parent are serialized on rename_lock. It's *NOT* something you want the stuff outside of core VFS to touch, though. It's a seqlock, and write_seqlock() on it is limited to fs/dcache.c alone. Reader side is allowed, but it's still not something you want to use lightly - outside of fs/dcache.c, fs/d_path.c and fs/namei.c there are only 3 users (ceph_mdsc_build_path(), nfs_path() and auditsc handle_path()). Don't add more without a discussion on fsdevel and detailed ACKs; it's quite likely that a better solution will be found.
With one exception (see the discussion of d_mark_tmpfile() in the end), ->d_name is also stabilized by that.
Slightly Less Blunt Tool You Still Should Not Use.
==================================================
->s_vfs_rename_mutex will stabilize ->d_parent. The trouble is, while it's not system-wide like rename_lock, it's fs-wide, so there's a plenty of contention to run into *AND* if you try that while ->i_rwsem is held on some directory in that filesystem, you are fucked - lock_rename() (and rename(2), and...) will deadlock on you. Nothing outside of fs/{namei,dcache}.c touches it directly; there is an indirect use (lock_rename()), but that should be done only around the call of cross-directory rename on _another_ filesystem - overlayfs moving stuff within the writable layer, ecryptfs doing rename on underlying filesystem, nfsd handling rename request from client, etc.
Anyone trying to use that hammer without a good reason will be very sorry - that's a promise. The pain will begin with the request to adjust the proof of deadlock avoidance in directory locking and it will only go downwards from there...
Parent's ->i_rwsem Held Exclusive.
==================================
That stabilizes ->d_parent and ->d_name. To be more precise,
holding parent->d_inode->i_rwsem exclusive stabilizes the result of (dentry->d_parent == parent). That is to say, dentry that is a child of parent will remain such and dentry that isn't a child won't become a child.
holding parent->d_inode->i_rwsem exclusive stabilizes ->d_name of all children. In other words, if you've locked the parent exclusive and found something to be its child while keeping the parent locked, child will have ->d_parent and ->d_name stable until you unlock the parent.
That covers most of the directory-modifying methods - stuff like ->mkdir(), ->unlink(), ->rename(), etc. can access ->d_parent and ->d_name of the dentry argument(s) without any extra locks; the caller is already holding enough. Well, unless you are special (*cough* apparmor *cough*) and feel like dropping and regaining the lock on parent inside your ->mkdir()... Don't do that, please - you might have no renames, but there's a plenty of other headache you can get into that way.
Negatives.
==========
Negative dentry doesn't change ->d_parent or ->d_name. Of course, that is only worth something if you are guaranteed that it won't become positive under you - if that happens, all bets are off.
Holding the inode of parent locked (at least) shared is enough to guarantee that. That takes care of ->lookup() instances - their dentry argument has ->d_parent and ->d_name stable until you make it positive (normally - by d_splice_alias()). Once you've done d_splice_alias(), you'd better be careful with access to those; you won't get hit by concurrent rename() (it locks parent(s) exclusive), but if your inode is a directory and d_splice_alias() *elsewhere* picks the same inode (fs image corruption, network filesystem with rename done from another client behind your back, etc.), you'll see the sucker moved.
In practice, d_splice_alias() is the last thing done by most of ->lookup() instances - whatever it has returned gets returned by ->lookup() itself, possibly after freeing some temporary allocations, etc. The rest needs to watch out for accesses to ->d_name and ->d_parent downstream of d_splice_alias() return.
Another case where we are guaranteed that dentry is negative and will stay so is ->d_release() - it's called for a dentry that is well on the way to becoming an ex-parrot; it's already marked dead, unhashed and negative. So a ->d_release() instance doesn't have to worry about ->d_name and ->d_parent - both are valid and stable.
sprintf().
==========
%pd prints dentry name, safely. %p2d - parent_name/dentry_name, etc. up to %p4d. %pD .. %p4D do the same by file reference. Any time you see pr_warn("Some weird bollocks with %s (%d)\n", dentry->d_name, err); it should've beenpr_warn("Some weird bollocks with %pd (%d)\n", dentry, err)...
d_path() and friends are there for purpose - don't open-code those without a damn good reason.
Checking if one dentry is an ancestor of another.
=================================================
Use the damn is_subdir(), don't open-code it.
Spinlocks.
==========
dentry->d_lock stabilizes ->d_parent and ->d_name (as well as almost everything else about dentry); downside is that it's a spinlock *and* nesting it is not to be attempted easily; you are allowed to lock child while holding lock on parent, but very few places have any business doing that (only 2 outside of VFS - tree-walking in autofs, which might eventually get redone avoiding that and fsnotify_set_children_dentry_flags(), which just might get moved to fs/dcache.c itself; we'll see)
Note that "almost everything" includes refcount; that is to say, dget() and dput() will spin if you are holding ->d_lock, so you can't dget() the parent under ->d_lock on child - that's a locking order violation that can easily deadlock on you. dget_parent() does that kind of thing safely, and a look at it might be instructive. Try to open-code something of that sort, and you'll be hurt.
Said that, dget_parent() is overused - it has legitimate uses, but more often than not it's the wrong tool. In particular, while you grab a reference to something that was the parent at some point during dget_parent(), it might not be the parent anymore by the time it is returned to caller.
Most of the dget_parent() uses are due to bad calling conventions of ->d_revalidate(). When that gets sanitized, those will be gone.
The methods that are called with ->d_lock get the protection - that would be ->d_delete() and ->d_prune().
->d_lock on parent is also sufficient; similar to exclusive ->i_rwsem, parent->d_lock stabilizes (dentry->d_parent == parent) and if child has been observed to have child->d_parent equal to parent after you've locked parent->d_lock, you know that child->d_{parent,name} will remain stable until you unlock the parent.
Name Snapshots.
===============
There's take_dentry_name_snapshot()/release_dentry_name_snapshot(). That stuff eats about 64 bytes on stack (longer names _are_ handled correctly; no allocations are needed, we can simply grab an extra reference to external name and hold it). Can't be done under ->d_lock, won't do anything about ->d_parent *and* there's nothing to prevent dentry being renamed while you are looking at the name snapshot. Sometimes it's useful...
RCU Headaches.
==============
See e.g. https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git/commit/?h=fixes.pathwalk-rcu-2&id=8d0a75eba81813cbb00beb73a67783e1cde9982f
(NB: ought to repost that)
[*] d_mark_tmpfile() is pretty much a delayed bit of constructor. There is a possible intermediate state of dentry ("will be tmpfile one"); dentries in that state are all created in vfs_tmpfile(), get passed to ->tmpfile() where they transition to normal unhashed postive dentries. The reason why name is not set from the very beginning is that at that point we do not know the inumber of inode they are going to get (that becomes known inside ->tmpfile() instance) and we want that inumber seen in their names (for /proc/*/fd/*, basically). Name change is done while holding ->d_lock both on dentry itself and on its parent.
Re: [RFC PATCH 19/24] erofs: introduce namei alternative to C
In reply to Al ViroQuoted 7 lines · Al Viro
Quoted 5 lines · Gao Xiang
Quoted 4 lines
Sorry for my ignorance.
I mean i just borrowed the code from thefs/erofs/namei.cand i directly translated that into Rust code. That might be a problem that also exists in original working C code.
Quoted 6 lines
Quoted 2 lines
As for EROFS (an immutable fs), I think after
d_splice_alias(),d_nameis still stable (since we don't have rename semantics likewise for now).Even on corrupted images? If you have two directories with entries that act as hardlinks to the same subdirectory, and keep hitting them on lookups, it will have to transplant the subtree between the parents.
Oh, I missed unexpected directory hardlink corrupted cases.
Quoted 5 lines
Quoted 3 lines
But as the generic filesystem POV,
d_nameaccess is actually tricky under RCU walk path indeed.->lookup() is never called in RCU mode.
I know, I just said d_name access is tricky in RCU walk.
->lookup() is for real lookup, not search dcache as fast cached lookup in the RCU context.
Thanks,
Gao Xiang
RFC PATCH 20/24 erofs: introduce readdir alternative to C
In reply to Yiyang WuThis patch introduce erofs_readdir_rust as an alternative for erofs_readdir written in Rust.
--- fs/erofs/Makefile | 2 +- fs/erofs/dir.c | 2 ++ fs/erofs/dir_rs.rs | 57 ++++++++++++++++++++++++++++++++++++++++ fs/erofs/internal.h | 1 + fs/erofs/rust_bindings.h | 1 + 5 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 fs/erofs/dir_rs.rs
fs/erofs/Makefile +1 -1
@@ -9,4 +9,4 @@ erofs-$(CONFIG_EROFS_FS_ZIP_DEFLATE) += decompressor_deflate.o erofs-$(CONFIG_EROFS_FS_ZIP_ZSTD) += decompressor_zstd.o erofs-$(CONFIG_EROFS_FS_BACKED_BY_FILE) += fileio.o erofs-$(CONFIG_EROFS_FS_ONDEMAND) += fscache.o -erofs-$(CONFIG_EROFS_FS_RUST) += super_rs.o inode_rs.o namei_rs.o rust_helpers.o +erofs-$(CONFIG_EROFS_FS_RUST) += super_rs.o inode_rs.o namei_rs.o dir_rs.o rust_helpers.o
fs/erofs/dir.c +2 -0
@@ -6,6 +6,7 @@*/ #include "internal.h" +#ifndef CONFIG_EROFS_FS_RUST static int erofs_fill_dentries (struct inode *dir,struct dir_context *ctx,void *dentry_blk,struct erofs_dirent *de,unsigned int nameoff0,unsigned int maxsize)
@@ -92,6 +93,7 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx)erofs_put_metabuf (&buf);return err <0 ? err :0 ; } +#endif const struct file_operations erofs_dir_fops = { .llseek = generic_file_llseek,
fs/erofs/dir_rs.rs new file +57 -0
57 lines not on this page · raw message
fs/erofs/internal.h +1 -0
@@ -444,6 +444,7 @@ int erofs_iget5_set(struct inode *inode, void *opaque);#define erofs_iget erofs_iget_rust#define erofs_get_parent erofs_get_parent_rust#define erofs_lookup erofs_lookup_rust +#define erofs_readdir erofs_readdir_rust#else struct inode *erofs_iget (struct super_block *sb,erofs_nid_t nid);#endif
fs/erofs/rust_bindings.h +1 -0
@@ -23,4 +23,5 @@ extern struct inode *erofs_iget_rust(struct super_block *sb, erofs_nid_t nid);extern struct dentry *erofs_lookup_rust (struct inode *inode,struct dentry *dentry,unsigned int flags);extern struct dentry *erofs_get_parent_rust (struct dentry *dentry); +extern int erofs_readdir_rust (struct file *file,struct dir_context *ctx);#endif
Signature · 1 line
2.46.0
RFC PATCH 21/24 erofs: introduce erofs_map_blocks alternative to C
In reply to Yiyang WuThis patch introduces erofs_map_blocks alternative written in Rust, which will be hooked inside the erofs_iomap_begin.
--- fs/erofs/Makefile | 2 +- fs/erofs/data.c | 5 ++++ fs/erofs/data_rs.rs | 63 ++++++++++++++++++++++++++++++++++++++++ fs/erofs/rust_bindings.h | 4 +++ 4 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 fs/erofs/data_rs.rs
fs/erofs/Makefile +1 -1
@@ -9,4 +9,4 @@ erofs-$(CONFIG_EROFS_FS_ZIP_DEFLATE) += decompressor_deflate.o erofs-$(CONFIG_EROFS_FS_ZIP_ZSTD) += decompressor_zstd.o erofs-$(CONFIG_EROFS_FS_BACKED_BY_FILE) += fileio.o erofs-$(CONFIG_EROFS_FS_ONDEMAND) += fscache.o -erofs-$(CONFIG_EROFS_FS_RUST) += super_rs.o inode_rs.o namei_rs.o dir_rs.o rust_helpers.o +erofs-$(CONFIG_EROFS_FS_RUST) += super_rs.o inode_rs.o namei_rs.o dir_rs.o data_rs.o rust_helpers.o
fs/erofs/data.c +5 -0
@@ -293,7 +293,12 @@ static int erofs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, map.m_la = offset; map.m_llen = length; +#ifdef CONFIG_EROFS_FS_RUST + ret =erofs_map_blocks_rust (inode, &map); +#else ret =erofs_map_blocks (inode, &map); +#endif +if (ret <0 )return ret;
fs/erofs/data_rs.rs new file +63 -0
63 lines not on this page · raw message
fs/erofs/rust_bindings.h +4 -0
@@ -24,4 +24,8 @@ extern struct dentry *erofs_lookup_rust(struct inode *inode, struct dentry *dent unsigned int flags);extern struct dentry *erofs_get_parent_rust (struct dentry *dentry);extern int erofs_readdir_rust (struct file *file,struct dir_context *ctx); + +struct erofs_map_blocks ; +extern int erofs_map_blocks_rust (struct inode *inode, +struct erofs_map_blocks *map);#endif
Signature · 1 line
2.46.0
RFC PATCH 22/24 erofs: add skippable iters in Rust
In reply to Yiyang WuThis patch introduce self-owned skippable data iterators in Rust. This iterators will be used to access extended attributes later.
--- fs/erofs/rust/erofs_sys/data/raw_iters.rs | 121 ++++++++++++++++++++++ 1 file changed, 121 insertions(+)
fs/erofs/rust/erofs_sys/data/raw_iters.rs +121 -0
@@ -4,3 +4,124 @@pub (crate )mod ref_iter;mod traits;pub (crate )use traits::*; + +use super ::*; +use alloc::boxed::Box ; + +/// Represents a skippable continuous buffer iterator. This is used primarily for reading the +/// extended attributes. Since the key-value is flattened out in its original format. +pub (crate )struct SkippableContinuousIter <'a> { + iter:Box <dyn ContinuousBufferIter <'a> + 'a>, + data:RefBuffer <'a>, + cur:usize , +} + +fn cmp_with_cursor_move ( + lhs: &[u8 ], + rhs: &[u8 ], + lhs_cur: &mut usize , + rhs_cur: &mut usize , + len:usize , +) ->bool { +let result = lhs[*lhs_cur..(*lhs_cur + len)] == rhs[*rhs_cur..(*rhs_cur + len)]; + *lhs_cur += len; + *rhs_cur += len; + result +} + +#[derive( Debug , Clone , Copy , PartialEq )] +pub (crate )enum SkipCmpError { +PosixError (Errno ), +NotEqual (Off ), +} + +impl From <Errno >for SkipCmpError { +fn from (e:Errno ) ->Self { +SkipCmpError ::PosixError (e) + } +} + +impl <'a>SkippableContinuousIter <'a> { +pub (crate )fn try_new ( +mut iter:Box <dyn ContinuousBufferIter <'a> + 'a>, + ) ->PosixResult <Option <Self >> { +if iter.eof () { +return Ok (None ); + } +let data = iter.next ().unwrap ()?; +Ok (Some (Self { iter, data, cur:0 })) + } +pub (crate )fn skip (&mut self, offset:Off ) ->PosixResult <()> { +let dlen = self.data.content ().len () - self.cur; +if offsetas usize <= dlen { + self.cur += offsetas usize ; + }else { + self.cur =0 ; + self.iter.advance_off (dlenas Off ); + self.data = self.iter.next ().unwrap ()?; + } +Ok (()) + } + +pub (crate )fn read (&mut self, buf: &mut [u8 ]) ->PosixResult <()> { +let mut dlen = self.data.content ().len () - self.cur; +let mut bcur =0_usize ; +let blen = buf.len (); +if dlen !=0 && dlen >= blen { + buf.clone_from_slice (&self.data.content ()[self.cur..(self.cur + blen)]); + self.cur += blen; + }else { + buf[bcur..(bcur + dlen)].copy_from_slice (&self.data.content ()[self.cur..]); + bcur += dlen; +while bcur < blen { + self.cur =0 ; + self.data = self.iter.next ().unwrap ()?; + dlen = self.data.content ().len (); +if dlen >= blen - bcur { + buf[bcur..].copy_from_slice (&self.data.content ()[..(blen - bcur)]); + self.cur = blen - bcur; +return Ok (()); + }else { + buf[bcur..(bcur + dlen)].copy_from_slice (self.data.content ()); + bcur += dlen; + } + } + } +Ok (()) + } + +pub (crate )fn try_cmp (&mut self, buf: &[u8 ]) ->Result <(),SkipCmpError > { +let dlen = self.data.content ().len () - self.cur; +let blen = buf.len (); +let mut bcur =0_usize ; + +if dlen !=0 && dlen >= blen { +if cmp_with_cursor_move (self.data.content (), buf, &mut self.cur, &mut bcur, blen) { +Ok (()) + }else { +Err (SkipCmpError ::NotEqual (bcuras Off )) + } + }else { +if dlen !=0 { +let clen = dlen.min (blen); +if !cmp_with_cursor_move (self.data.content (), buf, &mut self.cur, &mut bcur, clen) { +return Err (SkipCmpError ::NotEqual (bcuras Off )); + } + } +while bcur < blen { + self.cur =0 ; + self.data = self.iter.next ().unwrap ()?; +let dlen = self.data.content ().len (); +let clen = dlen.min (blen - bcur); +if !cmp_with_cursor_move (self.data.content (), buf, &mut self.cur, &mut bcur, clen) { +return Err (SkipCmpError ::NotEqual (bcuras Off )); + } + } + +Ok (()) + } + } +pub (crate )fn eof (&self) ->bool { + self.data.content ().len () - self.cur ==0 && self.iter.eof () + } +}
Signature · 1 line
2.46.0
RFC PATCH 23/24 erofs: implement xattrs operations in Rust
In reply to Yiyang WuThis patch adds xattrs for erofs_sys crate and will later be used to implement xattr handler in Rust.
--- fs/erofs/inode_rs.rs | 7 +- fs/erofs/rust/erofs_sys/inode.rs | 1 + fs/erofs/rust/erofs_sys/operations.rs | 27 ++++ fs/erofs/rust/erofs_sys/superblock.rs | 141 +++++++++++++++++++++ fs/erofs/rust/erofs_sys/superblock/mem.rs | 13 +- fs/erofs/rust/erofs_sys/xattrs.rs | 148 ++++++++++++++++++++++ fs/erofs/rust/kinode.rs | 6 + 7 files changed, 341 insertions(+), 2 deletions(-)
fs/erofs/inode_rs.rs +6 -1
@@ -48,8 +48,13 @@ fn try_fill_inode(k_inode: NonNull<inode>, nid: Nid) -> PosixResult<()> {let erofs_inode: &mut KernelInode =unsafe { &mut *(container_of! (k_inode.as_ptr (),KernelInode , k_inode)as *mut KernelInode ) }; - erofs_inode.info.write (sbi.filesystem.read_inode_info (nid)?); +let info = sbi.filesystem.read_inode_info (nid)?; erofs_inode.nid.write (nid); + erofs_inode.shared_entries.write ( + sbi.filesystem + .read_inode_xattrs_shared_entries (nid, &info)?, + ); + erofs_inode.info.write (info);Ok (()) }/// Exported as fill_inode additional fill inode
fs/erofs/rust/erofs_sys/inode.rs +1 -0
@@ -299,6 +299,7 @@ pub(crate) trait Inode: Sized {fn new (_sb: &SuperBlock , info:InodeInfo , nid:Nid ) ->Self ;fn info (&self) -> &InodeInfo ;fn nid (&self) ->Nid ; +fn xattrs_shared_entries (&self) -> &XAttrSharedEntries ; }/// Represents the error which occurs when trying to convert the inode.
fs/erofs/rust/erofs_sys/operations.rs +27 -0
@@ -1,9 +1,16 @@// Copyright 2024 Yiyang Wu // SPDX-License-Identifier: MIT or GPL-2.0-or-later +use super ::alloc_helper::*; +use super ::data::raw_iters::*; +use super ::data::*;use super ::inode::*;use super ::superblock::*; +use super ::xattrs::*;use super ::*; +use alloc::vec::Vec ; + +use crate ::round;pub (crate )fn read_inode <'a,I ,C >( filesystem: &'a dyn FileSystem<I>,
@@ -33,3 +40,23 @@ pub(crate) fn dir_lookup<'a, I, C>(read_inode (filesystem, collection, nid) }) } + +pub (crate )fn get_xattr_infixes <'a>( + iter: &mut (dyn ContinuousBufferIter <'a> + 'a), +) ->PosixResult <Vec <XAttrInfix >> { +let mut result:Vec <XAttrInfix > =Vec ::new (); +for datain iter { +let buffer = data?; +let buf = buffer.content (); +let len = buf.len (); +let mut cur:usize =0 ; +while cur <= len { +let mut infix:Vec <u8 > =Vec ::new (); +let size = u16::from_le_bytes ([buf[cur], buf[cur +1 ]])as usize ; +extend_from_slice (&mut infix, &buf[cur +2 ..cur +2 + size])?; +push_vec (&mut result,XAttrInfix (infix))?; + cur =round! (UP , cur +2 + size,4 ); + } + } +Ok (result) +}
fs/erofs/rust/erofs_sys/superblock.rs +141 -0
161 lines not on this page · raw message
fs/erofs/rust/erofs_sys/superblock/mem.rs +12 -1
@@ -1,8 +1,8 @@// Copyright 2024 Yiyang Wu // SPDX-License-Identifier: MIT or GPL-2.0-or-later -use super ::alloc_helper::*;use super ::data::raw_iters::ref_iter::*; +use super ::operations::*;use super ::*;// Memory Mapped Device/File so we need to have some external lifetime on the backend trait.
@@ -16,6 +16,7 @@ pub(crate) struct KernelFileSystem<B> backend: B, sb: SuperBlock, device_info: DeviceInfo, + infixes: Vec<XAttrInfix>, }impl <I ,B >FileSystem <I >for KernelFileSystem <B >
@@ -58,6 +59,9 @@ fn continuous_iter<'a>(fn device_info (&self) -> &DeviceInfo { &self.device_info } +fn xattr_infixes (&self) -> &Vec <XAttrInfix > { + &self.infixes + } }impl <B >KernelFileSystem <B >
@@ -68,6 +72,12 @@ pub(crate) fn try_new(backend: B) -> PosixResult<Self> {let mut buf =SUPERBLOCK_EMPTY_BUF ; backend.fill (&mut buf,EROFS_SUPER_OFFSET )?;let sb:SuperBlock = buf.into (); +let infixes =get_xattr_infixes (&mut ContinuousRefIter ::new ( + &sb, + &backend, + sb.xattr_prefix_startas Off , + sb.xattr_prefix_countas Off *4 , + ))?; let device_info = get_device_infos(&mut ContinuousRefIter::new( &sb, &backend,
@@ -78,6 +88,7 @@ pub(crate) fn try_new(backend: B) -> PosixResult<Self> { backend, sb, device_info, + infixes, }) } }
fs/erofs/rust/erofs_sys/xattrs.rs +148 -0
158 lines not on this page · raw message
fs/erofs/rust/kinode.rs +6 -0
@@ -11,6 +11,7 @@use super ::erofs_sys::errnos::*;use super ::erofs_sys::inode::*;use super ::erofs_sys::superblock::*; +use super ::erofs_sys::xattrs::*;use super ::erofs_sys::*;extern "C" {
@@ -22,6 +23,7 @@pub (crate )struct KernelInode {pub (crate ) info:MaybeUninit <InodeInfo >,pub (crate ) nid:MaybeUninit <Nid >, +pub (crate ) shared_entries:MaybeUninit <XAttrSharedEntries >,pub (crate ) k_inode:MaybeUninit <inode >,pub (crate ) k_opaque:MaybeUninit <*mut c_void >, }
@@ -31,6 +33,7 @@ fn new(_sb: &SuperBlock, _info: InodeInfo, _nid: Nid) -> Self {Self { info:MaybeUninit ::uninit (), nid:MaybeUninit ::uninit (), + shared_entries:MaybeUninit ::uninit (), k_inode:MaybeUninit ::uninit (), k_opaque:MaybeUninit ::uninit (), }
@@ -41,6 +44,9 @@ fn nid(&self) -> Nid {fn info (&self) -> &InodeInfo {unsafe { self.info.assume_init_ref () } } +fn xattrs_shared_entries (&self) -> &XAttrSharedEntries { +unsafe { self.shared_entries.assume_init_ref () } + } }pub (crate )struct KernelInodeCollection {
Signature · 1 line
2.46.0
RFC PATCH 24/24 erofs: introduce xattrs replacement to C
In reply to Yiyang WuThis patch introduces erofs_getxattr_rust and erofs_listxattr_rust to C and can replace the original xattr logic entirely.
Note that the original acl implementation is tweaked with a lifted function called erofs_getxattr_nobuf, so that difference of the calling convention of Rust side code can be bridged.
--- fs/erofs/Makefile | 3 ++ fs/erofs/rust_bindings.h | 8 +++ fs/erofs/xattr.c | 31 +++++++++--- fs/erofs/xattr.h | 7 +++ fs/erofs/xattr_rs.rs | 106 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 148 insertions(+), 7 deletions(-) create mode 100644 fs/erofs/xattr_rs.rs
fs/erofs/Makefile +3 -0
@@ -10,3 +10,6 @@ erofs-$(CONFIG_EROFS_FS_ZIP_ZSTD) += decompressor_zstd.o erofs-$(CONFIG_EROFS_FS_BACKED_BY_FILE) += fileio.o erofs-$(CONFIG_EROFS_FS_ONDEMAND) += fscache.o erofs-$(CONFIG_EROFS_FS_RUST) += super_rs.o inode_rs.o namei_rs.o dir_rs.o data_rs.o rust_helpers.o +ifeq ($(CONFIG_EROFS_FS_XATTR),y) +erofs-$(CONFIG_EROFS_FS_RUST) += xattr_rs.o +endif
fs/erofs/rust_bindings.h +8 -0
@@ -28,4 +28,12 @@ extern int erofs_readdir_rust(struct file *file, struct dir_context *ctx);struct erofs_map_blocks ;extern int erofs_map_blocks_rust (struct inode *inode,struct erofs_map_blocks *map); +extern int erofs_getxattr_rust (struct inode *inode,unsigned int flags, +const char *name,void *buffer,size_t size); +extern ssize_t erofs_listxattr_rust (struct dentry *dentry,char *buffer, +size_t buffer_size); +#ifdef CONFIG_EROFS_FS_POSIX_ACL +extern int erofs_getxattr_nobuf_rust (struct inode *inode,int prefix, +const char *name,char **value); +#endif #endif
fs/erofs/xattr.c +24 -7
@@ -8,6 +8,7 @@#include <linux/xxhash.h> #include "xattr.h" +#ifndef CONFIG_EROFS_FS_RUST struct erofs_xattr_iter { struct super_block *sb; struct erofs_buf buf;
@@ -122,6 +123,7 @@ static int erofs_init_inode_xattrs(struct inode *inode)clear_and_wake_up_bit (EROFS_I_BL_XATTR_BIT , &vi->flags); return ret; } +#endif static bool erofs_xattr_user_list (struct dentry *dentry) {
@@ -175,6 +177,7 @@ const struct xattr_handler * const erofs_xattr_handlers[] = { NULL, }; +#ifndef CONFIG_EROFS_FS_RUST static int erofs_xattr_copy_to_buffer (struct erofs_xattr_iter *it,unsigned int len) {
@@ -509,8 +512,28 @@ int erofs_xattr_prefixes_init(struct super_block *sb)erofs_xattr_prefixes_cleanup (sb);return ret; } +#endif #ifdef CONFIG_EROFS_FS_POSIX_ACL +#ifndef CONFIG_EROFS_FS_RUST +static int erofs_getxattr_nobuf (struct inode *inode,int prefix, +const char *name,char **value) +{ +int rc; +char *buf =NULL ; + rc =erofs_getxattr (inode, prefix, name,NULL ,0 ); +if (rc >0 ) { + buf =kmalloc (rc,GFP_KERNEL ); +if (!value) +return ENOMEM ; + rc =erofs_getxattr (inode, prefix, name, buf, rc); + } + *value = buf; +return rc; +} +#else +#define erofs_getxattr_nobuf erofs_getxattr_nobuf_rust +#endif struct posix_acl *erofs_get_acl (struct inode *inode,int type,bool rcu) {struct posix_acl *acl;
@@ -531,13 +554,7 @@ struct posix_acl *erofs_get_acl(struct inode *inode, int type, bool rcu)return ERR_PTR (-EINVAL ); } - rc =erofs_getxattr (inode, prefix,"" ,NULL ,0 ); -if (rc >0 ) { - value =kmalloc (rc,GFP_KERNEL ); -if (!value) -return ERR_PTR (-ENOMEM ); - rc =erofs_getxattr (inode, prefix,"" , value, rc); - } + rc =erofs_getxattr_nobuf (inode, prefix,"" , &value);if (rc == -ENOATTR ) acl =NULL ;
fs/erofs/xattr.h +7 -0
@@ -46,10 +46,17 @@ static inline const char *erofs_xattr_prefix(unsigned int idx,extern const struct xattr_handler *const erofs_xattr_handlers[]; +#ifdef CONFIG_EROFS_FS_RUST +#define erofs_getxattr erofs_getxattr_rust +#define erofs_listxattr erofs_listxattr_rust +static inline int erofs_xattr_prefixes_init (struct super_block *sb) {return 0 ; } +static inline void erofs_xattr_prefixes_cleanup (struct super_block *sb) {} +#else int erofs_xattr_prefixes_init (struct super_block *sb);void erofs_xattr_prefixes_cleanup (struct super_block *sb);int erofs_getxattr (struct inode *,int ,const char *,void *,size_t );ssize_t erofs_listxattr (struct dentry *,char *,size_t ); +#endif #else static inline int erofs_xattr_prefixes_init (struct super_block *sb) {return 0 ; }static inline void erofs_xattr_prefixes_cleanup (struct super_block *sb) {}
fs/erofs/xattr_rs.rs new file +106 -0
106 lines not on this page · raw message
Signature · 1 line
2.46.0