fs: cache the string generated by reading /proc/filesystems
Subsystems (3)
PATCH v2 1/1 fs: cache the string generated by reading /proc/filesystems
It is being read surprisingly often (e.g., by mkdir, ls and even sed!).
This is lock-protected pointer chasing over a linked list to pay for sprintf for every fs (32 on my boxen).
Instead cache the result.
open+read+close cycle single-threaded (ops/s):
before: 442732
after: 1063462 (+140%)
Here the main bottleneck is memcg.
Scalability-wise problems are avoidable lockref trip on open and ref management for the file on procfs side.
The file looks like a sterotypical C from the 90s, right down to an open-coded and slightly obfuscated linked list. I intentionally did not clean up any of it -- I think the file will be best served by a Rust rewrite when the time comes.
---
v2:
- drop the procfs bits
- touch up some comments
I posted v1 last year https://lore.kernel.org/linux-fsdevel/[email protected]/
but that ran into some procfs issues. the thing can be sped up regardless of the procfs problem.
fs/filesystems.c | 144 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 137 insertions(+), 7 deletions(-)
fs/filesystems.c +137 -7
@@ -34,6 +34,23 @@static struct file_system_type *file_systems;static DEFINE_RWLOCK (file_systems_lock); +#ifdef CONFIG_PROC_FS +static unsigned long file_systems_gen; + +struct file_systems_string { +struct rcu_head rcufree; +unsigned long gen; +size_t len; +char string[]; +}; +static struct file_systems_string *file_systems_string; +static void invalidate_filesystems_string (void ); +#else +static void invalidate_filesystems_string (void ) +{ +} +#endif +/* WARNING: This can be used only if we _already_ own a reference */ struct file_system_type *get_filesystem (struct file_system_type *fs) {
@@ -83,10 +100,12 @@ int register_filesystem(struct file_system_type * fs)return -EBUSY ;write_lock (&file_systems_lock); p =find_filesystem (fs->name,strlen (fs->name)); -if (*p) +if (*p) { res = -EBUSY ; -else + }else { *p = fs; +invalidate_filesystems_string (); + }write_unlock (&file_systems_lock);return res; }
@@ -115,6 +134,7 @@ int unregister_filesystem(struct file_system_type * fs)if (fs == *tmp) { *tmp = fs->next; fs->next =NULL ; +invalidate_filesystems_string ();write_unlock (&file_systems_lock);synchronize_rcu ();return 0 ;
@@ -235,22 +255,132 @@ int __init list_bdev_fs_names(char *buf, size_t size) }#ifdef CONFIG_PROC_FS -static int filesystems_proc_show (struct seq_file *m,void *v) +/* +* The fs list gets queried a lot by userspace because of libselinux, including +* rather surprising programs (would you guess *sed* is on the list?). In order +* to reduce the overhead we cache the resulting string, which normally hangs +* around below 512 bytes in size. +* +* As the list almost never changes, its creation is not particularly optimized +* for simplicity. +* +* We sort it out on read in order to not introduce a failure point for fs +* registration (in principle we may be unable to alloc memory for the list). +*/ +static void invalidate_filesystems_string (void ) { -struct file_system_type * tmp; +struct file_systems_string *fss; -read_lock (&file_systems_lock); +lockdep_assert_held_write (&file_systems_lock); + file_systems_gen++; + fss = file_systems_string; +WRITE_ONCE (file_systems_string,NULL ); +kfree_rcu (fss, rcufree); +} + +static noinline int regen_filesystems_string(void) +{ +struct file_system_type *tmp; +struct file_systems_string *old, *new; +size_t newlen, usedlen; +unsigned long gen; + +retry: +lockdep_assert_not_held (&file_systems_lock); + + newlen =0 ; +write_lock (&file_systems_lock); + gen = file_systems_gen; + tmp = file_systems; + +/* pre-calc space for "%s\t%s\n" for each fs */ +while (tmp) { +if (!(tmp->fs_flags &FS_REQUIRES_DEV )) + newlen +=strlen ("nodev" ); + newlen +=strlen ("\t" ); + newlen +=strlen (tmp->name); + newlen +=strlen ("\n" ); + tmp = tmp->next; + } +write_unlock (&file_systems_lock); + + new =kmalloc (offsetof(struct file_systems_string , string) + newlen +1 , +GFP_KERNEL ); +if (!new) +return -ENOMEM ; + + new->gen = gen; + new->len = newlen; + new->string[newlen] ='\0' ; +write_lock (&file_systems_lock); + old = file_systems_string; + +/* +* Did someone beat us to it? +*/ +if (old && old->gen == file_systems_gen) { +write_unlock (&file_systems_lock); +kfree (new); +return 0 ; + } + +/* +* Did the list change in the meantime? +*/ +if (gen != file_systems_gen) { +write_unlock (&file_systems_lock); +kfree (new); + goto retry; + } + +/* +* Populate the string. +* +* We know we have just enough space because we calculated the right +* size the previous time we had the lock and confirmed the list has +* not changed after reacquiring it. +*/ + usedlen =0 ; tmp = file_systems;while (tmp) { -seq_printf (m,"%s\t%s\n" , + usedlen +=sprintf (&new->string[usedlen],"%s\t%s\n" , (tmp->fs_flags &FS_REQUIRES_DEV ) ?"" :"nodev" , tmp->name); tmp = tmp->next; } -read_unlock (&file_systems_lock); +BUG_ON (new->len !=strlen (new->string)); + +/* +* Paired with consume fence in READ_ONCE() in filesystems_proc_show() +*/ +smp_store_release (&file_systems_string, new); +write_unlock (&file_systems_lock); +kfree_rcu (old, rcufree);return 0 ; } +static int filesystems_proc_show (struct seq_file *m,void *v) +{ +struct file_systems_string *fss; + +for (;;) { +scoped_guard (rcu) { +/* +* Paired with smp_store_release() in regen_filesystems_string() +*/ + fss =READ_ONCE (file_systems_string); +if (likely (fss)) { +seq_write (m, fss->string, fss->len); +return 0 ; + } + } + +int err =regen_filesystems_string (); +if (unlikely (err)) +return err; + } +} +static int __init proc_filesystems_init (void ) {proc_create_single ("filesystems" ,0 ,NULL , filesystems_proc_show);
Signature · 1 line
2.48.1
Re: [PATCH v2] cache the string generated by reading /proc/filesystems
Quoted 2 lines · Mateusz Guzik
It is being read surprisingly often (e.g., by mkdir, ls and even sed!).
Yeah, it is sad that bloat continually grows to continually make fast systems slow.
Quoted 56 lines
This is lock-protected pointer chasing over a linked list to pay for sprintf for every fs (32 on my boxen).
Instead cache the result.
open+read+close cycle single-threaded (ops/s):
before: 442732
after: 1063462 (+140%)Here the main bottleneck is memcg.
Scalability-wise problems are avoidable lockref trip on open and ref management for the file on procfs side.
The file looks like a sterotypical C from the 90s, right down to an open-coded and slightly obfuscated linked list. I intentionally did not clean up any of it -- I think the file will be best served by a Rust rewrite when the time comes.
Signed-off-by Mateusz Guzik <[email protected]>---
v2:
- drop the procfs bits
- touch up some commentsI posted v1 last year https://lore.kernel.org/linux-fsdevel/[email protected]/
but that ran into some procfs issues. the thing can be sped up regardless of the procfs problem.
fs/filesystems.c| 144 ++++++++++++++++++++++++++++++++++++++++++++---1 file changed, 137 insertions(+), 7 deletions(-)fs/filesystems.c +20
-0+static noinline int regen_filesystems_string(void) +{ +struct file_system_type *tmp; +struct file_systems_string *old, *new; +size_t newlen, usedlen; +unsigned long gen; + +retry: +lockdep_assert_not_held (&file_systems_lock); + + newlen =0 ; +write_lock (&file_systems_lock); + gen = file_systems_gen; + tmp = file_systems; + +/* pre-calc space for "%s\t%s\n" for each fs */ +while (tmp) { +if (!(tmp->fs_flags &FS_REQUIRES_DEV )) + newlen +=strlen ("nodev" ); + newlen +=strlen ("\t" );
fs/filesystems.c int __init list_bdev_fs_names(char *buf, size_t size) · line 310
Quoted 10 lines
fs/filesystems.c +10
-0+ newlen +=strlen (tmp->name); + newlen +=strlen ("\n" ); + tmp = tmp->next; + } +write_unlock (&file_systems_lock); + + new =kmalloc (offsetof(struct file_systems_string , string) + newlen +1 , +GFP_KERNEL ); +if (!new) +return -ENOMEM ;
This is introducing an error case where there was none before, in a critical system file. It would also be possible to fall back to seq_printf() instead of sprinft() if no buffer is available.
Quoted 21 lines
fs/filesystems.c +21
-0+ new->gen = gen; + new->len = newlen; + new->string[newlen] ='\0' ; +write_lock (&file_systems_lock); + old = file_systems_string; + +/* +* Did someone beat us to it? +*/ +if (old && old->gen == file_systems_gen) { +write_unlock (&file_systems_lock); +kfree (new); +return 0 ; + } + +/* +* Did the list change in the meantime? +*/ +if (gen != file_systems_gen) { +write_unlock (&file_systems_lock); +kfree (new);
fs/filesystems.c int __init list_bdev_fs_names(char *buf, size_t size) · line 342
Quoted 10 lines
fs/filesystems.c +10
-0+ goto retry; + } + +/* +* Populate the string. +* +* We know we have just enough space because we calculated the right +* size the previous time we had the lock and confirmed the list has +* not changed after reacquiring it. +*/
It doesn't seem prudent to use raw sprintf() in a loop and justify this in a comment, when the actual string generation is not performance critical. This could use scnprintf() and be "safer" even if it "should just work" (tm).
fs/filesystems.c int __init list_bdev_fs_names(char *buf, size_t size) · line 351
Quoted 11 lines
fs/filesystems.c +3
-2+ usedlen =0 ; tmp = file_systems;while (tmp) { -seq_printf (m,"%s\t%s\n" , + usedlen +=sprintf (&new->string[usedlen],"%s\t%s\n" , (tmp->fs_flags &FS_REQUIRES_DEV ) ?"" :"nodev" , tmp->name); tmp = tmp->next; } -read_unlock (&file_systems_lock); +BUG_ON (new->len !=strlen (new->string));
Adding a BUG_ON() here seems gratuitous, when there are lots of other options to handle this case, especially if sprintf() didn't just corrupt memory for some reason. It could allocate a larger buffer and retry, or in the worst case return an error instead of crashing the system.
fs/filesystems.c int __init list_bdev_fs_names(char *buf, size_t size) · line 360
Quoted 10 lines
fs/filesystems.c +7
-0+ +/* +* Paired with consume fence in READ_ONCE() in filesystems_proc_show() +*/ +smp_store_release (&file_systems_string, new); +write_unlock (&file_systems_lock); +kfree_rcu (old, rcufree);return 0 ; }
Cheers, Andreas
Al Viro
maintainerRe: [PATCH v2] cache the string generated by reading /proc/filesystems
Quoted 6 lines · Andreas Dilger
Quoted 2 lines · Mateusz Guzik
It is being read surprisingly often (e.g., by mkdir, ls and even sed!).
Yeah, it is sad that bloat continually grows to continually make fast systems slow.
libselinux looking for selinuxfs, at a guess, not that I had any idea why sed might possibly care about that...
<RTFS>
-i support, apparently.
No printable comment...
Re: [PATCH v2] cache the string generated by reading /proc/filesystems
In reply to Andreas DilgerQuoted 11 lines · Andreas Dilger
fs/filesystems.c int __init list_bdev_fs_names(char *buf, size_t size) · line 310
Quoted 4 lines · Mateusz Guzik
fs/filesystems.c +4
-0+ new =kmalloc (offsetof(struct file_systems_string , string) + newlen +1 , +GFP_KERNEL ); +if (!new) +return -ENOMEM ;This is introducing an error case where there was none before, in a critical system file. It would also be possible to fall back to
seq_printf()instead ofsprinft()if no buffer is available.
Fair, should be easy to do.
Quoted 8 lines
fs/filesystems.c int __init list_bdev_fs_names(char *buf, size_t size) · line 342
Quoted 7 lines
fs/filesystems.c +7
-0+/* +* Populate the string. +* +* We know we have just enough space because we calculated the right +* size the previous time we had the lock and confirmed the list has +* not changed after reacquiring it. +*/
Quoted 4 lines
It doesn't seem prudent to use raw
sprintf()in a loop and justify this in a comment, when the actual string generation is not performance critical. This could usescnprintf()and be "safer" even if it "should just work" (tm).
I don't get how that's supposed help anything. I still have to call the routine in a loop, except this time I have to keep recalculating remaining size by hand on top of it.
What this really wants is a string library which handles easy things like this one, FreeBSD has something like this in
https://man.freebsd.org/cgi/man.cgi?sbuf(9) , but I certainly can't be bothered to work on an equivalent or porting it over.
If one is to "overflow-proof this" with total disregard for performance, a series of strlcats would do it.
Quoted 12 lines
fs/filesystems.c int __init list_bdev_fs_names(char *buf, size_t size) · line 351
Quoted 11 lines
fs/filesystems.c +3
-2+ usedlen =0 ; tmp = file_systems;while (tmp) { -seq_printf (m,"%s\t%s\n" , + usedlen +=sprintf (&new->string[usedlen],"%s\t%s\n" , (tmp->fs_flags &FS_REQUIRES_DEV ) ?"" :"nodev" , tmp->name); tmp = tmp->next; } -read_unlock (&file_systems_lock); +BUG_ON (new->len !=strlen (new->string));
Quoted 5 lines
Adding a
BUG_ON()here seems gratuitous, when there are lots of other options to handle this case, especially ifsprintf()didn't just corrupt memory for some reason. It could allocate a larger buffer and retry, or in the worst case return an error instead of crashing the system.
Well nobody is supposed to get the splat, except maybe someone messing with how the string is generated, so I don't really see your point -- it's purely a bug-catching measure.
However, it can still just print a warn and fallback to the current pointer chasing.
Christian Brauner
maintainerQuoted 16 lines · Mateusz Guzik
It is being read surprisingly often (e.g., by mkdir, ls and even sed!).
This is lock-protected pointer chasing over a linked list to pay for sprintf for every fs (32 on my boxen).
Instead cache the result.
open+read+close cycle single-threaded (ops/s):
before: 442732
after: 1063462 (+140%)Here the main bottleneck is memcg.
Scalability-wise problems are avoidable lockref trip on open and ref management for the file on procfs side.
Quoted 7 lines
The file looks like a sterotypical C from the 90s, right down to an open-coded and slightly obfuscated linked list. I intentionally did not clean up any of it -- I think the file will be best served by a Rust rewrite when the time comes.
Signed-off-by Mateusz Guzik <[email protected]>---
Ugh, can't we at least also get rid of the rwlock stuff? Something like the appended two (completely untested) patches.
From 148061e07d8dfc513df6d190791811ed98e21adc Mon Sep 17 00:00:00 2001
From: Mateusz Guzik <[email protected]>
Date: Sat, 29 Mar 2025 20:28:20 +0100
Subject: [PATCH 1/2] proc: add a helper for marking files as permanent by
external consumers
This avoids refing trips on use, added with VFS in mind but should be helpful elsewhere as well.
--- fs/proc/generic.c | 6 ++++++ include/linux/proc_fs.h | 1 + 2 files changed, 7 insertions(+)
fs/proc/generic.c +6 -0
@@ -677,6 +677,12 @@ struct proc_dir_entry *proc_create_single_data(const char *name, umode_t mode, }EXPORT_SYMBOL (proc_create_single_data); +void proc_make_permanent (struct proc_dir_entry *de) +{ +pde_make_permanent (de); +} +EXPORT_SYMBOL (proc_make_permanent); +void proc_set_size (struct proc_dir_entry *de,loff_t size) { de->size = size;
include/linux/proc_fs.h +1 -0
@@ -106,6 +106,7 @@ struct proc_dir_entry *proc_create_single_data(const char *name, umode_t mode, int (*show)(struct seq_file *, void *), void *data);#define proc_create_single (name, mode, parent, show) \ proc_create_single_data(name, mode, parent, show, NULL) +void proc_make_permanent (struct proc_dir_entry *); extern struct proc_dir_entry *proc_create_data(const char *, umode_t, struct proc_dir_entry *,
Signature · 1 line
2.47.3
From 5c5a25c5ab66f666349e80f78b5168a9fb4eebd6 Mon Sep 17 00:00:00 2001
From: Christian Brauner <[email protected]>
Date: Thu, 23 Apr 2026 14:43:17 +0200
Subject: [PATCH 2/2] fs: RCU-ify filesystems list and cache /proc/filesystems
The drivers list was protected by an rwlock; every mount, every open of /proc/filesystems and the legacy sysfs(2) syscall walked a hand-rolled singly-linked list under it. /proc/filesystems is especially hot because libselinux causes programs as mundane as mkdir, ls and sed to open and read it on every invocation.
Convert the list to an RCU-protected hlist and switch the writer side to a plain spinlock. Writers keep their existing non-sleeping section while readers walk under rcu_read_lock() with no lock traffic:
- register_filesystem()/unregister_filesystem() take file_systems_lock, publish via hlist_{add_tail,del_init}_rcu() and invalidate the cached /proc/filesystems string. unregister_filesystem() keeps its synchronize_rcu() after dropping the lock so in-flight readers are drained before the module (and its embedded file_system_type) can go away.
- __get_fs_type(), list_bdev_fs_names() and the fs_index()/fs_name()/fs_maxindex() helpers walk the list under rcu_read_lock(). fs_name() continues to drop the read-side lock after try_module_get() and accesses ->name outside the RCU section; the module reference pins the embedded file_system_type across the boundary.
Additionally cache the pre-rendered "[nodev\t]<name>\n" blob so filesystems_proc_show() becomes a single seq_write() under rcu_read_lock(). On a cache miss regen_filesystems_string() sizes the buffer under RCU, allocates GFP_KERNEL and refills under the writer spinlock; a generation counter bumped under the spinlock lets it detect concurrent (un)register and retry the allocation.
Finally mark /proc/filesystems as permanent so the per-open refcount bounce (which can degrade to a spinlock acquire) is removed; this was the dominant bottleneck once the list walk itself was off the critical path and is what lifts the multi-threaded numbers in Mateusz Guzik's measurements:
https://lore.kernel.org/linux-fsdevel/[email protected]/
struct file_system_type::next becomes struct hlist_node list; no in-tree caller references the old ->next field outside fs/filesystems.c.
--- fs/filesystems.c | 282 +++++++++++++++++++++++++++++---------------- fs/ocfs2/super.c | 1 - include/linux/fs.h | 2 +- 3 files changed, 181 insertions(+), 104 deletions(-)
fs/filesystems.c +180 -102
392 lines not on this page · raw message
fs/ocfs2/super.c +0 -1
@@ -1224,7 +1224,6 @@ static struct file_system_type ocfs2_fs_type = { .name = "ocfs2", .kill_sb = kill_block_super, .fs_flags = FS_REQUIRES_DEV|FS_RENAME_DOES_D_MOVE, - .next = NULL, .init_fs_context = ocfs2_init_fs_context, .parameters = ocfs2_param_spec, };
include/linux/fs.h +1 -1
@@ -2286,7 +2286,7 @@ struct file_system_type {const struct fs_parameter_spec *parameters;void (*kill_sb) (struct super_block *);struct module *owner; -struct file_system_type * next; +struct hlist_node list;struct hlist_head fs_supers;struct lock_class_key s_lock_key;
Signature · 1 line
2.47.3
Quoted 23 lines · Christian Brauner
Quoted 21 lines · Mateusz Guzik
It is being read surprisingly often (e.g., by mkdir, ls and even sed!).
This is lock-protected pointer chasing over a linked list to pay for sprintf for every fs (32 on my boxen).
Instead cache the result.
open+read+close cycle single-threaded (ops/s):
before: 442732 after: 1063462 (+140%)Here the main bottleneck is memcg.
Scalability-wise problems are avoidable lockref trip on open and ref management for the file on procfs side.
The file looks like a sterotypical C from the 90s, right down to an open-coded and slightly obfuscated linked list. I intentionally did not clean up any of it -- I think the file will be best served by a Rust rewrite when the time comes.
Quoted 5 lines
Quoted 2 lines
Signed-off-by Mateusz Guzik <[email protected]>---
Ugh, can't we at least also get rid of the rwlock stuff? Something like the appended two (completely untested) patches.
The procfs thing got protest last time, see
https://lore.kernel.org/all/CACVxJT_eXRNjp-2sEfuxYmzTMPvu7U1R2bsKYjadVfR-W691og@mail.gmail.com/
Alexey wrote his own variant and put my name on it:
https://lore.kernel.org/linux-fsdevel/c58291cd-0775-4c90-8443-ba71897b5cbb@p183/
. I'm not going to protest the patch, but I'm not signing off on the thing. Perhaps you can pull it while removing my credit? The author cc'ed.
The wtf list handling cleanup + rwlock removal should certainly be their own commit. applying all of this on top of my patch should be a matter of few shell commands, alternatively I can massage this the other way around.
I did not go for the other changes because a big chunk of the file is the suspicious sysv syscall marked for removal and which I have 0 intentions of testing.
Finallly, a review did not like the BUG_ON (which you replaced with WARN_ON_ONCE) and the new failure mode of ENOMEM. Both bogus calculation and ENOMEM can be handled with a fallback to iterating the list like it is done now.
I can do the massaging if it sounds like a plan.
Christian Brauner
maintainerQuoted 48 lines · Mateusz Guzik
Quoted 23 lines · Christian Brauner
Quoted 21 lines · Mateusz Guzik
It is being read surprisingly often (e.g., by mkdir, ls and even sed!).
This is lock-protected pointer chasing over a linked list to pay for sprintf for every fs (32 on my boxen).
Instead cache the result.
open+read+close cycle single-threaded (ops/s):
before: 442732 after: 1063462 (+140%)Here the main bottleneck is memcg.
Scalability-wise problems are avoidable lockref trip on open and ref management for the file on procfs side.
The file looks like a sterotypical C from the 90s, right down to an open-coded and slightly obfuscated linked list. I intentionally did not clean up any of it -- I think the file will be best served by a Rust rewrite when the time comes.
Quoted 5 lines
Quoted 2 lines
Signed-off-by Mateusz Guzik <[email protected]>---
Ugh, can't we at least also get rid of the rwlock stuff? Something like the appended two (completely untested) patches.
The procfs thing got protest last time, see
https://lore.kernel.org/all/CACVxJT_eXRNjp-2sEfuxYmzTMPvu7U1R2bsKYjadVfR-W691og@mail.gmail.com/Alexey wrote his own variant and put my name on it:
https://lore.kernel.org/linux-fsdevel/c58291cd-0775-4c90-8443-ba71897b5cbb@p183/
. I'm not going to protest the patch, but I'm not signing off on the thing. Perhaps you can pull it while removing my credit? The author cc'ed.The wtf list handling cleanup + rwlock removal should certainly be their own commit. applying all of this on top of my patch should be a matter of few shell commands, alternatively I can massage this the other way around.
I did not go for the other changes because a big chunk of the file is the suspicious sysv syscall marked for removal and which I have 0 intentions of testing.
Quoted 6 lines
Finallly, a review did not like the
BUG_ON(which you replaced withWARN_ON_ONCE) and the new failure mode of ENOMEM. Both bogus calculation and ENOMEM can be handled with a fallback to iterating the list like it is done now.I can do the massaging if it sounds like a plan.
Yeah, sounds good to me.