fs/file.c stuff
Subsystems (23)
Al Viro
maintainerPATCH 1/1 fs/file.c stuff
Assorted cleanups, part from the previous cycle, part new.
Branch is in git.kernel.org:/pub/scm/linux/kernel/git/viro/vfs.git #work.fdtable Individual patches in followups.
Appears to work; if nobody objects, into -next it goes...
Shortlog:
Al Viro (8):
get rid of ...lookup...fdget_rcu() family
remove pointless includes of <linux/fdtable.h>
close_files(): don't bother with xchg()
proc_fd_getattr(): don't bother with S_ISDIR() check
move close_range(2) into fs/file.c, fold __close_range() into it sane_fdtable_size(): don't bother looking at descriptors we are not going to copy alloc_fdtable(): change calling conventions.
dup_fd(): change calling conventions
Yu Ma (3):
fs/file.c: remove sanity_check and add likely/unlikely in alloc_fd() fs/file.c: conditionally clear full_fds
fs/file.c: add fast path in find_next_fd()
Diffstat:
arch/powerpc/platforms/cell/spufs/coredump.c | 4 +- fs/fcntl.c | 1 - fs/file.c | 195 +++++++++++---------------- fs/file_table.c | 1 - fs/gfs2/glock.c | 12 +- fs/notify/dnotify/dnotify.c | 5 +- fs/notify/fanotify/fanotify.c | 1 - fs/notify/fanotify/fanotify_user.c | 1 - fs/open.c | 17 --- fs/overlayfs/copy_up.c | 1 - fs/proc/base.c | 1 - fs/proc/fd.c | 23 +--- include/linux/fdtable.h | 7 +- include/linux/file.h | 1 + io_uring/io_uring.c | 1 - kernel/bpf/bpf_inode_storage.c | 1 - kernel/bpf/bpf_task_storage.c | 1 - kernel/bpf/task_iter.c | 6 +- kernel/bpf/token.c | 1 - kernel/exit.c | 1 - kernel/fork.c | 26 ++-- kernel/kcmp.c | 4 +- kernel/module/dups.c | 1 - kernel/module/kmod.c | 1 - kernel/umh.c | 1 - net/handshake/request.c | 1 - security/apparmor/domain.c | 1 - 27 files changed, 104 insertions(+), 212 deletions(-)
Al Viro
maintainerPATCH 1/11 get rid of ...lookup...fdget_rcu() family
Once upon a time, predecessors of those used to do file lookup without bumping a refcount, provided that caller held rcu_read_lock() across the lookup and whatever it wanted to read from the struct file found. When struct file allocation switched to SLAB_TYPESAFE_BY_RCU, that stopped being feasible and these primitives started to bump the file refcount for lookup result, requiring the caller to call fput() afterwards.
But that turned them pointless - e.g.
rcu_read_lock(); file = lookup_fdget_rcu(fd); rcu_read_unlock(); is equivalent to file = fget_raw(fd);
and all callers of lookup_fdget_rcu() are of that form. Similarly, task_lookup_fdget_rcu() calls can be replaced with calling fget_task(). task_lookup_next_fdget_rcu() doesn't have direct counterparts, but its callers would be happier if we replaced it with an analogue that deals with RCU internally.
--- arch/powerpc/platforms/cell/spufs/coredump.c | 4 +-- fs/file.c | 28 +++----------------- fs/gfs2/glock.c | 12 ++------- fs/notify/dnotify/dnotify.c | 5 +--- fs/proc/fd.c | 12 +++------ include/linux/fdtable.h | 4 --- include/linux/file.h | 1 + kernel/bpf/task_iter.c | 6 +---- kernel/kcmp.c | 4 +-- 9 files changed, 14 insertions(+), 62 deletions(-)
arch/powerpc/platforms/cell/spufs/coredump.c +1 -3
@@ -73,9 +73,7 @@ static struct spu_context *coredump_next_context(int *fd)return NULL ; *fd = n -1 ; -rcu_read_lock (); - file =lookup_fdget_rcu (*fd); -rcu_read_unlock (); + file =fget_raw (*fd);if (file) { ctx =SPUFS_I (file_inode (file))->i_ctx;get_spu_context (ctx);
fs/file.c +4 -24
@@ -1064,29 +1064,7 @@ struct file *fget_task(struct task_struct *task, unsigned int fd)return file; } -struct file *lookup_fdget_rcu (unsigned int fd) -{ -return __fget_files_rcu (current->files, fd,0 ); - -} -EXPORT_SYMBOL_GPL (lookup_fdget_rcu); - -struct file *task_lookup_fdget_rcu (struct task_struct *task,unsigned int fd) -{ -/* Must be called with rcu_read_lock held */ -struct files_struct *files; -struct file *file =NULL ; - -task_lock (task); - files = task->files; -if (files) - file =__fget_files_rcu (files, fd,0 ); -task_unlock (task); - -return file; -} - -struct file *task_lookup_next_fdget_rcu (struct task_struct *task,unsigned int *ret_fd) +struct file *fget_task_next (struct task_struct *task,unsigned int *ret_fd) {/* Must be called with rcu_read_lock held */ struct files_struct *files;
@@ -1096,17 +1074,19 @@ struct file *task_lookup_next_fdget_rcu(struct task_struct *task, unsigned int *task_lock (task); files = task->files;if (files) { +rcu_read_lock ();for (; fd <files_fdtable (files)->max_fds; fd++) { file =__fget_files_rcu (files, fd,0 );if (file)break ; } +rcu_read_unlock (); }task_unlock (task); *ret_fd = fd;return file; } -EXPORT_SYMBOL (task_lookup_next_fdget_rcu); +EXPORT_SYMBOL (fget_task_next);/* * Lightweight file lookup - no refcnt increment if fd table isn't shared.
fs/gfs2/glock.c +2 -10
@@ -34,7 +34,6 @@#include <linux/lockref.h> #include <linux/rhashtable.h> #include <linux/pid_namespace.h> -#include <linux/fdtable.h> #include <linux/file.h> #include "gfs2.h"
@@ -2765,25 +2764,18 @@ static struct file *gfs2_glockfd_next_file(struct gfs2_glockfd_iter *i) i->file =NULL ; } -rcu_read_lock ();for (;; i->fd++) { -struct inode *inode; - - i->file =task_lookup_next_fdget_rcu (i->task, &i->fd); + i->file =fget_task_next (i->task, &i->fd);if (!i->file) { i->fd =0 ;break ; } - inode =file_inode (i->file); -if (inode->i_sb == i->sb) +if (file_inode (i->file)->i_sb == i->sb)break ; -rcu_read_unlock ();fput (i->file); -rcu_read_lock (); } -rcu_read_unlock ();return i->file; }
fs/notify/dnotify/dnotify.c +1 -4
@@ -16,7 +16,6 @@#include <linux/security.h> #include <linux/spinlock.h> #include <linux/slab.h> -#include <linux/fdtable.h> #include <linux/fsnotify_backend.h> static int dir_notify_enable__read_mostly =1 ;
@@ -343,9 +342,7 @@ int fcntl_dirnotify(int fd, struct file *filp, unsigned int arg) new_fsn_mark =NULL ; } -rcu_read_lock (); - f =lookup_fdget_rcu (fd); -rcu_read_unlock (); + f =fget_raw (fd);/* if (f != filp) means that we lost a race and another task/thread * actually closed the fd we are still playing with before we grabbed
fs/proc/fd.c +3 -9
@@ -116,9 +116,7 @@ static bool tid_fd_mode(struct task_struct *task, unsigned fd, fmode_t *mode) {struct file *file; -rcu_read_lock (); - file =task_lookup_fdget_rcu (task, fd); -rcu_read_unlock (); + file =fget_task (task, fd);if (file) { *mode = file->f_mode;fput (file);
@@ -258,19 +256,17 @@ static int proc_readfd_common(struct file *file, struct dir_context *ctx,if (!dir_emit_dots (file, ctx)) goto out; -rcu_read_lock ();for (fd = ctx->pos -2 ;; fd++) {struct file *f;struct fd_data data;char name[10 +1 ];unsigned int len; - f =task_lookup_next_fdget_rcu (p, &fd); + f =fget_task_next (p, &fd); ctx->pos = fd +2LL ;if (!f)break ; data.mode = f->f_mode; -rcu_read_unlock ();fput (f); data.fd = fd;
@@ -278,11 +274,9 @@ static int proc_readfd_common(struct file *file, struct dir_context *ctx,if (!proc_fill_cache (file, ctx, name, len, instantiate, p, &data)) - goto out; +break ;cond_resched (); -rcu_read_lock (); } -rcu_read_unlock (); out:put_task_struct (p);return 0 ;
include/linux/fdtable.h +0 -4
@@ -93,10 +93,6 @@ static inline struct file *files_lookup_fd_locked(struct files_struct *files, unreturn files_lookup_fd_raw (files, fd); } -struct file *lookup_fdget_rcu (unsigned int fd); -struct file *task_lookup_fdget_rcu (struct task_struct *task,unsigned int fd); -struct file *task_lookup_next_fdget_rcu (struct task_struct *task,unsigned int *fd); -static inline bool close_on_exec (unsigned int fd,const struct files_struct *files) {return test_bit (fd,files_fdtable (files)->close_on_exec);
include/linux/file.h +1 -0
@@ -51,6 +51,7 @@ static inline void fdput(struct fd fd)extern struct file *fget (unsigned int fd);extern struct file *fget_raw (unsigned int fd);extern struct file *fget_task (struct task_struct *task,unsigned int fd); +extern struct file *fget_task_next (struct task_struct *task,unsigned int *fd);extern unsigned long __fdget (unsigned int fd);extern unsigned long __fdget_raw (unsigned int fd);extern unsigned long __fdget_pos (unsigned int fd);
kernel/bpf/task_iter.c +1 -5
@@ -5,7 +5,6 @@#include <linux/namei.h> #include <linux/pid_namespace.h> #include <linux/fs.h> -#include <linux/fdtable.h> #include <linux/filter.h> #include <linux/bpf_mem_alloc.h> #include <linux/btf_ids.h>
@@ -286,17 +285,14 @@ task_file_seq_get_next(struct bpf_iter_seq_task_file_info *info) curr_fd =0 ; } -rcu_read_lock (); - f =task_lookup_next_fdget_rcu (curr_task, &curr_fd); + f =fget_task_next (curr_task, &curr_fd);if (f) {/* set info->fd */ info->fd = curr_fd; -rcu_read_unlock ();return f; }/* the current task is done, go to the next task */ -rcu_read_unlock ();put_task_struct (curr_task);if (info->common.type ==BPF_TASK_ITER_TID ) {
kernel/kcmp.c +1 -3
@@ -63,9 +63,7 @@ get_file_raw_ptr(struct task_struct *task, unsigned int idx) {struct file *file; -rcu_read_lock (); - file =task_lookup_fdget_rcu (task, idx); -rcu_read_unlock (); + file =fget_task (task, idx);if (file)fput (file);
Signature · 1 line
2.39.2
Al Viro
maintainerPATCH 2/11 remove pointless includes of <linux/fdtable.h>
some of those used to be needed, some had been cargo-culted for no reason...
--- fs/fcntl.c | 1 - fs/file_table.c | 1 - fs/notify/fanotify/fanotify.c | 1 - fs/notify/fanotify/fanotify_user.c | 1 - fs/overlayfs/copy_up.c | 1 - fs/proc/base.c | 1 - io_uring/io_uring.c | 1 - kernel/bpf/bpf_inode_storage.c | 1 - kernel/bpf/bpf_task_storage.c | 1 - kernel/bpf/token.c | 1 - kernel/exit.c | 1 - kernel/module/dups.c | 1 - kernel/module/kmod.c | 1 - kernel/umh.c | 1 - net/handshake/request.c | 1 - security/apparmor/domain.c | 1 - 16 files changed, 16 deletions(-)
fs/fcntl.c +0 -1
@@ -12,7 +12,6 @@#include <linux/fs.h> #include <linux/filelock.h> #include <linux/file.h> -#include <linux/fdtable.h> #include <linux/capability.h> #include <linux/dnotify.h> #include <linux/slab.h>
fs/file_table.c +0 -1
@@ -9,7 +9,6 @@#include <linux/string.h> #include <linux/slab.h> #include <linux/file.h> -#include <linux/fdtable.h> #include <linux/init.h> #include <linux/module.h> #include <linux/fs.h>
fs/notify/fanotify/fanotify.c +0 -1
@@ -1,6 +1,5 @@// SPDX-License-Identifier: GPL-2.0 #include <linux/fanotify.h> -#include <linux/fdtable.h> #include <linux/fsnotify_backend.h> #include <linux/init.h> #include <linux/jiffies.h>
fs/notify/fanotify/fanotify_user.c +0 -1
@@ -1,7 +1,6 @@// SPDX-License-Identifier: GPL-2.0 #include <linux/fanotify.h> #include <linux/fcntl.h> -#include <linux/fdtable.h> #include <linux/file.h> #include <linux/fs.h> #include <linux/anon_inodes.h>
fs/overlayfs/copy_up.c +0 -1
@@ -16,7 +16,6 @@#include <linux/sched/signal.h> #include <linux/cred.h> #include <linux/namei.h> -#include <linux/fdtable.h> #include <linux/ratelimit.h> #include <linux/exportfs.h> #include "overlayfs.h"
fs/proc/base.c +0 -1
@@ -58,7 +58,6 @@#include <linux/init.h> #include <linux/capability.h> #include <linux/file.h> -#include <linux/fdtable.h> #include <linux/generic-radix-tree.h> #include <linux/string.h> #include <linux/seq_file.h>
io_uring/io_uring.c +0 -1
@@ -51,7 +51,6 @@#include <linux/sched/signal.h> #include <linux/fs.h> #include <linux/file.h> -#include <linux/fdtable.h> #include <linux/mm.h> #include <linux/mman.h> #include <linux/percpu.h>
kernel/bpf/bpf_inode_storage.c +0 -1
@@ -16,7 +16,6 @@#include <uapi/linux/btf.h> #include <linux/bpf_lsm.h> #include <linux/btf_ids.h> -#include <linux/fdtable.h> #include <linux/rcupdate_trace.h> DEFINE_BPF_STORAGE_CACHE (inode_cache);
kernel/bpf/bpf_task_storage.c +0 -1
@@ -16,7 +16,6 @@#include <linux/filter.h> #include <uapi/linux/btf.h> #include <linux/btf_ids.h> -#include <linux/fdtable.h> #include <linux/rcupdate_trace.h> DEFINE_BPF_STORAGE_CACHE (task_cache);
kernel/bpf/token.c +0 -1
@@ -1,6 +1,5 @@#include <linux/bpf.h> #include <linux/vmalloc.h> -#include <linux/fdtable.h> #include <linux/file.h> #include <linux/fs.h> #include <linux/kernel.h>
kernel/exit.c +0 -1
@@ -25,7 +25,6 @@#include <linux/acct.h> #include <linux/tsacct_kern.h> #include <linux/file.h> -#include <linux/fdtable.h> #include <linux/freezer.h> #include <linux/binfmts.h> #include <linux/nsproxy.h>
kernel/module/dups.c +0 -1
@@ -18,7 +18,6 @@#include <linux/completion.h> #include <linux/cred.h> #include <linux/file.h> -#include <linux/fdtable.h> #include <linux/workqueue.h> #include <linux/security.h> #include <linux/mount.h>
kernel/module/kmod.c +0 -1
@@ -15,7 +15,6 @@#include <linux/completion.h> #include <linux/cred.h> #include <linux/file.h> -#include <linux/fdtable.h> #include <linux/workqueue.h> #include <linux/security.h> #include <linux/mount.h>
kernel/umh.c +0 -1
@@ -13,7 +13,6 @@#include <linux/completion.h> #include <linux/cred.h> #include <linux/file.h> -#include <linux/fdtable.h> #include <linux/fs_struct.h> #include <linux/workqueue.h> #include <linux/security.h>
net/handshake/request.c +0 -1
@@ -13,7 +13,6 @@#include <linux/module.h> #include <linux/skbuff.h> #include <linux/inet.h> -#include <linux/fdtable.h> #include <linux/rhashtable.h> #include <net/sock.h>
security/apparmor/domain.c +0 -1
@@ -9,7 +9,6 @@*/ #include <linux/errno.h> -#include <linux/fdtable.h> #include <linux/fs.h> #include <linux/file.h> #include <linux/mount.h>
Signature · 1 line
2.39.2
Al Viro
maintainerPATCH 3/11 close_files(): don't bother with xchg()
In reply to Al ViroAt that point nobody else has references to the victim files_struct; as the matter of fact, the caller will free it immediately after close_files() returns, with no RCU delays or anything of that sort.
That's why we are not protecting against fdtable reallocation on expansion, not cleaning the bitmaps, etc. There's no point zeroing the pointers in ->fd[] either, let alone make that an atomic operation.
--- fs/file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
fs/file.c +1 -1
@@ -428,7 +428,7 @@ static struct fdtable *close_files(struct files_struct * files) set = fdt->open_fds[j++];while (set) {if (set &1 ) { -struct file * file =xchg (&fdt->fd[i],NULL ); +struct file *file = fdt->fd[i];if (file) {filp_close (file, files);cond_resched ();
Signature · 1 line
2.39.2
Al Viro
maintainerPATCH 4/11 proc_fd_getattr(): don't bother with S_ISDIR() check
In reply to Al Virothat thing is callable only as ->i_op->getattr() instance and only for directory inodes (/proc/*/fd and /proc/*/task/*/fd)
--- fs/proc/fd.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-)
fs/proc/fd.c +1 -10
@@ -351,18 +351,9 @@ static int proc_fd_getattr(struct mnt_idmap *idmap, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); -int rv =0 ;generic_fillattr (&nop_mnt_idmap, request_mask, inode, stat); - -/* If it's a directory, put the number of open fds there */ -if (S_ISDIR (inode->i_mode)) { - rv =proc_readfd_count (inode, &stat->size); -if (rv <0 ) -return rv; - } - -return rv; +return proc_readfd_count (inode, &stat->size); } const struct inode_operations proc_fd_inode_operations = {
Signature · 1 line
2.39.2
Al Viro
maintainerPATCH 5/11 move close_range(2) into fs/file.c, fold __close_range() into it
In reply to Al Viro We never had callers for __close_range() except for close_range(2)
itself. Nothing of that sort has appeared in four years and if any users do show up, we can always separate those suckers again.
--- fs/file.c | 6 ++++-- fs/open.c | 17 ----------------- include/linux/fdtable.h | 1 - 3 files changed, 4 insertions(+), 20 deletions(-)
fs/file.c +4 -2
@@ -728,7 +728,7 @@ static inline void __range_close(struct files_struct *files, unsigned int fd, }/** -* __close_range() - Close all file descriptors in a given range. +* sys_close_range() - Close all file descriptors in a given range. * * @fd: starting file descriptor to close * @max_fd: last file descriptor to close
@@ -736,8 +736,10 @@ static inline void __range_close(struct files_struct *files, unsigned int fd,* * This closes a range of file descriptors. All file descriptors * from @fd up to and including @max_fd are closed. +* Currently, errors to close a given file descriptor are ignored. */ -int __close_range (unsigned fd,unsigned max_fd,unsigned int flags) +SYSCALL_DEFINE3(close_range, unsigned int, fd, unsigned int, max_fd, + unsigned int, flags) {struct task_struct *me = current;struct files_struct *cur_fds = me->files, *fds =NULL ;
fs/open.c +0 -17
@@ -1575,23 +1575,6 @@ SYSCALL_DEFINE1(close, unsigned int, fd)return retval; } -/** -* sys_close_range() - Close all file descriptors in a given range. -* -* @fd: starting file descriptor to close -* @max_fd: last file descriptor to close -* @flags: reserved for future extensions -* -* This closes a range of file descriptors. All file descriptors -* from @fd up to and including @max_fd are closed. -* Currently, errors to close a given file descriptor are ignored. -*/ -SYSCALL_DEFINE3(close_range, unsigned int, fd, unsigned int, max_fd, - unsigned int, flags) -{ -return __close_range (fd, max_fd, flags); -} -/* * This routine simulates a hangup on the tty, to arrange that users * are given clean terminals at login time.
include/linux/fdtable.h +0 -1
@@ -109,7 +109,6 @@ int iterate_fd(struct files_struct *, unsigned, const void *);extern int close_fd (unsigned int fd); -extern int __close_range (unsigned int fd,unsigned int max_fd,unsigned int flags);extern struct file *file_close_fd (unsigned int fd);extern int unshare_fd (unsigned long unshare_flags,unsigned int max_fds,struct files_struct **new_fdp);
Signature · 1 line
2.39.2
Al Viro
maintainerPATCH 6/11 sane_fdtable_size(): don't bother looking at descriptors we are not going to copy
In reply to Al Virowhen given a max_fds argument lower than that current size (that can happen when called from close_range(..., CLOSE_RANGE_UNSHARE)), we can ignore all descriptors >= max_fds.
--- fs/file.c | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-)
fs/file.c +23 -20
@@ -272,20 +272,6 @@ static inline bool fd_is_open(unsigned int fd, const struct fdtable *fdt)return test_bit (fd, fdt->open_fds); } -static unsigned int count_open_files (struct fdtable *fdt) -{ -unsigned int size = fdt->max_fds; -unsigned int i; - -/* Find the last open fd */ -for (i = size /BITS_PER_LONG ; i >0 ; ) { -if (fdt->open_fds[--i]) -break ; - } - i = (i +1 ) *BITS_PER_LONG ; -return i; -} -/* * Note that a sane fdtable size always has to be a multiple of * BITS_PER_LONG, since we have bitmaps that are sized by this.
@@ -297,16 +283,33 @@ static unsigned int count_open_files(struct fdtable *fdt)* * Rather than make close_range() have to worry about this, * just make that BITS_PER_LONG alignment be part of a sane -* fdtable size. Becuase that's really what it is. +* fdtable size. Because that's really what it is. */ static unsigned int sane_fdtable_size (struct fdtable *fdt,unsigned int max_fds) { -unsigned int count; +const unsigned int min_words =BITS_TO_LONGS (NR_OPEN_DEFAULT );// 1 +unsigned long mask; +unsigned int words; + +if (max_fds > fdt->max_fds) + max_fds = fdt->max_fds; + +if (max_fds ==NR_OPEN_DEFAULT ) +return NR_OPEN_DEFAULT ; + +/* +* What follows is a simplified find_last_bit(). There's no point +* finding exact last bit, when we are going to round it up anyway. +*/ + words =BITS_TO_LONGS (max_fds); + mask =BITMAP_LAST_WORD_MASK (max_fds); + +while (words > min_words && !(fdt->open_fds[words -1 ] & mask)) { + mask = ~0UL ; + words--; + } - count =count_open_files (fdt); -if (max_fds <NR_OPEN_DEFAULT ) - max_fds =NR_OPEN_DEFAULT ; -return ALIGN (min (count, max_fds),BITS_PER_LONG ); +return words *BITS_PER_LONG ; }/*
Signature · 1 line
2.39.2
Al Viro
maintainerPATCH 7/11 fs/file.c: remove sanity_check and add likely/unlikely in alloc_fd()
In reply to Al ViroFrom: Yu Ma <[email protected]>
alloc_fd() has a sanity check inside to make sure the struct file mapping to the allocated fd is NULL. Remove this sanity check since it can be assured by exisitng zero initilization and NULL set when recycling fd. Meanwhile, add likely/unlikely and expand_file() call avoidance to reduce the work under file_lock.
--- fs/file.c | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-)
fs/file.c +14 -19
@@ -514,7 +514,7 @@ static int alloc_fd(unsigned start, unsigned end, unsigned flags)if (fd < files->next_fd) fd = files->next_fd; -if (fd < fdt->max_fds) +if (likely (fd < fdt->max_fds)) fd =find_next_fd (fdt, fd);/*
@@ -522,19 +522,21 @@ static int alloc_fd(unsigned start, unsigned end, unsigned flags)* will limit the total number of files that can be opened. */ error = -EMFILE ; -if (fd >= end) +if (unlikely (fd >= end)) goto out; - error =expand_files (files, fd); -if (error <0 ) - goto out; +if (unlikely (fd >= fdt->max_fds)) { + error =expand_files (files, fd); +if (error <0 ) + goto out; -/* -* If we needed to expand the fs array we -* might have blocked - try again. -*/ -if (error) - goto repeat; +/* +* If we needed to expand the fs array we +* might have blocked - try again. +*/ +if (error) + goto repeat; + }if (start <= files->next_fd) files->next_fd = fd +1 ;
@@ -545,13 +547,6 @@ static int alloc_fd(unsigned start, unsigned end, unsigned flags)else __clear_close_on_exec (fd, fdt); error = fd; -#if 1 -/* Sanity check */ -if (rcu_access_pointer (fdt->fd[fd]) !=NULL ) { -printk (KERN_WARNING "alloc_fd: slot %d not NULL!\n" , fd); -rcu_assign_pointer (fdt->fd[fd],NULL ); - } -#endif out:spin_unlock (&files->file_lock);
@@ -617,7 +612,7 @@ void fd_install(unsigned int fd, struct file *file)rcu_read_unlock_sched ();spin_lock (&files->file_lock); fdt =files_fdtable (files); -BUG_ON (fdt->fd[fd] !=NULL ); +WARN_ON (fdt->fd[fd] !=NULL );rcu_assign_pointer (fdt->fd[fd], file);spin_unlock (&files->file_lock);return ;
Signature · 1 line
2.39.2
Al Viro
maintainerPATCH 8/11 fs/file.c: conditionally clear full_fds
In reply to Al ViroFrom: Yu Ma <[email protected]>
64 bits in open_fds are mapped to a common bit in full_fds_bits. It is very likely that a bit in full_fds_bits has been cleared before in __clear_open_fds()'s operation. Check the clear bit in full_fds_bits before clearing to avoid unnecessary write and cache bouncing. See commit fc90888d07b8 ("vfs: conditionally clear close-on-exec flag") for a similar optimization. take stock kernel with patch 1 as baseline, it improves pts/blogbench-1.1.0 read for 13%, and write for 5% on Intel ICX 160 cores configuration with v6.10-rc7.
--- fs/file.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
fs/file.c +3 -1
@@ -264,7 +264,9 @@ static inline void __set_open_fd(unsigned int fd, struct fdtable *fdt)static inline void __clear_open_fd (unsigned int fd,struct fdtable *fdt) {__clear_bit (fd, fdt->open_fds); -__clear_bit (fd /BITS_PER_LONG , fdt->full_fds_bits); + fd /=BITS_PER_LONG ; +if (test_bit (fd, fdt->full_fds_bits)) +__clear_bit (fd, fdt->full_fds_bits); }static inline bool fd_is_open (unsigned int fd,const struct fdtable *fdt)
Signature · 1 line
2.39.2
Al Viro
maintainerPATCH 9/11 fs/file.c: add fast path in find_next_fd()
In reply to Al ViroFrom: Yu Ma <[email protected]>
Skip 2-levels searching via find_next_zero_bit() when there is free slot in the word contains next_fd, as:
(1) next_fd indicates the lower bound for the first free fd.
(2) There is fast path inside of find_next_zero_bit() when size<=64 to speed up searching.
(3) After fdt is expanded (the bitmap size doubled for each time of expansion), it would never be shrunk. The search size increases but there are few open fds available here.
This fast path is proposed by Mateusz Guzik <[email protected]>, and agreed by Jan Kara <[email protected]>, which is more generic and scalable than previous versions. And on top of patch 1 and 2, it improves pts/blogbench-1.1.0 read by 8% and write by 4% on Intel ICX 160 cores configuration with v6.10-rc7.
--- fs/file.c | 9 +++++++++ 1 file changed, 9 insertions(+)
fs/file.c +9 -0
@@ -490,6 +490,15 @@ static unsigned int find_next_fd(struct fdtable *fdt, unsigned int start)unsigned int maxfd = fdt->max_fds;/* always multiple of BITS_PER_LONG */ unsigned int maxbit = maxfd /BITS_PER_LONG ;unsigned int bitbit = start /BITS_PER_LONG ; +unsigned int bit; + +/* +* Try to avoid looking at the second level bitmap +*/ + bit =find_next_zero_bit (&fdt->open_fds[bitbit],BITS_PER_LONG , + start & (BITS_PER_LONG -1 )); +if (bit <BITS_PER_LONG ) +return bit + bitbit *BITS_PER_LONG ; bitbit =find_next_zero_bit (fdt->full_fds_bits, maxbit, bitbit) *BITS_PER_LONG ;if (bitbit >= maxfd)
Signature · 1 line
2.39.2
Al Viro
maintainerPATCH 10/11 alloc_fdtable(): change calling conventions.
In reply to Al ViroFirst of all, tell it how many slots do we want, not which slot is wanted. It makes one caller (dup_fd()) more straightforward and doesn't harm another (expand_fdtable()).
Furthermore, make it return ERR_PTR() on failure rather than returning NULL. Simplifies the callers.
Simplify the size calculation, while we are at it - note that we always have slots_wanted greater than BITS_PER_LONG. What the rules boil down to is
* use the smallest power of two large enough to give us
that many slots
* on 32bit skip 64 and 128 - the minimal capacity we want
there is 256 slots (i.e. 1Kb fd array).
* on 64bit don't skip anything, the minimal capacity is
128 - and we'll never be asked for 64 or less. 128 slots means 1Kb fd array, again.
* on 128bit, if that ever happens, don't skip anything -
we'll never be asked for 128 or less, so the fd array allocation will be at least 2Kb.
--- fs/file.c | 58 ++++++++++++++++++------------------------------------- 1 file changed, 19 insertions(+), 39 deletions(-)
fs/file.c +19 -39
@@ -89,18 +89,11 @@ static void copy_fdtable(struct fdtable *nfdt, struct fdtable *ofdt)* 'unsigned long' in some places, but simply because that is how the Linux * kernel bitmaps are defined to work: they are not "bits in an array of bytes", * they are very much "bits in an array of unsigned long". -* -* The ALIGN(nr, BITS_PER_LONG) here is for clarity: since we just multiplied -* by that "1024/sizeof(ptr)" before, we already know there are sufficient -* clear low bits. Clang seems to realize that, gcc ends up being confused. -* -* On a 128-bit machine, the ALIGN() would actually matter. In the meantime, -* let's consider it documentation (and maybe a test-case for gcc to improve -* its code generation ;) */ -static struct fdtable *alloc_fdtable (unsigned int nr) +static struct fdtable *alloc_fdtable (unsigned int slots_wanted) {struct fdtable *fdt; +unsigned int nr;void *data;/*
@@ -110,20 +103,22 @@ static struct fdtable * alloc_fdtable(unsigned int nr)* the fdarray into comfortable page-tuned chunks: starting at 1024B * and growing in powers of two from there on. */ - nr /= (1024 /sizeof (struct file *)); - nr =roundup_pow_of_two (nr +1 ); - nr *= (1024 /sizeof (struct file *)); - nr =ALIGN (nr,BITS_PER_LONG ); +if (IS_ENABLED (CONFIG_32BIT ) && slots_wanted <256 ) + nr =256 ; +else + nr =roundup_pow_of_two (slots_wanted);/* * Note that this can drive nr *below* what we had passed if sysctl_nr_open -* had been set lower between the check in expand_files() and here. Deal -* with that in caller, it's cheaper that way. +* had been set lower between the check in expand_files() and here. * * We make sure that nr remains a multiple of BITS_PER_LONG - otherwise * bitmaps handling below becomes unpleasant, to put it mildly... */ -if (unlikely (nr > sysctl_nr_open)) - nr = ((sysctl_nr_open -1 ) | (BITS_PER_LONG -1 )) +1 ; +if (unlikely (nr > sysctl_nr_open)) { + nr =round_down (sysctl_nr_open,BITS_PER_LONG ); +if (nr < slots_wanted) +return ERR_PTR (-EMFILE ); + } fdt =kmalloc (sizeof (struct fdtable ),GFP_KERNEL_ACCOUNT );if (!fdt)
@@ -152,7 +147,7 @@ static struct fdtable * alloc_fdtable(unsigned int nr) out_fdt:kfree (fdt); out: -return NULL ; +return ERR_PTR (-ENOMEM ); }/*
@@ -169,7 +164,7 @@ static int expand_fdtable(struct files_struct *files, unsigned int nr)struct fdtable *new_fdt, *cur_fdt;spin_unlock (&files->file_lock); - new_fdt =alloc_fdtable (nr); + new_fdt =alloc_fdtable (nr +1 );/* make sure all fd_install() have seen resize_in_progress * or have finished their rcu_read_lock_sched() section.
@@ -178,16 +173,8 @@ static int expand_fdtable(struct files_struct *files, unsigned int nr)synchronize_rcu ();spin_lock (&files->file_lock); -if (!new_fdt) -return -ENOMEM ; -/* -* extremely unlikely race - sysctl_nr_open decreased between the check in -* caller and alloc_fdtable(). Cheaper to catch it here... -*/ -if (unlikely (new_fdt->max_fds <= nr)) { -__free_fdtable (new_fdt); -return -EMFILE ; - } +if (IS_ERR (new_fdt)) +return PTR_ERR (new_fdt); cur_fdt =files_fdtable (files);BUG_ON (nr < cur_fdt->max_fds);copy_fdtable (new_fdt, cur_fdt);
@@ -357,16 +344,9 @@ struct files_struct *dup_fd(struct files_struct *oldf, unsigned int max_fds, intif (new_fdt != &newf->fdtab)__free_fdtable (new_fdt); - new_fdt =alloc_fdtable (open_files -1 ); -if (!new_fdt) { - *errorp = -ENOMEM ; - goto out_release; - } - -/* beyond sysctl_nr_open; nothing to do */ -if (unlikely (new_fdt->max_fds < open_files)) { -__free_fdtable (new_fdt); - *errorp = -EMFILE ; + new_fdt =alloc_fdtable (open_files); +if (IS_ERR (new_fdt)) { + *errorp =PTR_ERR (new_fdt); goto out_release; }
Signature · 1 line
2.39.2
Al Viro
maintainerPATCH 11/11 dup_fd(): change calling conventions
In reply to Al Viroreturn ERR_PTR() on failure, get rid of errorp
--- fs/file.c | 14 ++++---------- include/linux/fdtable.h | 2 +- kernel/fork.c | 26 ++++++++++++-------------- 3 files changed, 17 insertions(+), 25 deletions(-)
fs/file.c +4 -10
@@ -306,17 +306,16 @@ static unsigned int sane_fdtable_size(struct fdtable *fdt, unsigned int max_fds)* passed in files structure. * errorp will be valid only when the returned files_struct is NULL. */ -struct files_struct *dup_fd (struct files_struct *oldf,unsigned int max_fds,int *errorp) +struct files_struct *dup_fd (struct files_struct *oldf,unsigned int max_fds) {struct files_struct *newf;struct file **old_fds, **new_fds;unsigned int open_files, i;struct fdtable *old_fdt, *new_fdt; - *errorp = -ENOMEM ; newf =kmem_cache_alloc (files_cachep,GFP_KERNEL );if (!newf) - goto out; +return ERR_PTR (-ENOMEM );atomic_set (&newf->count,1 );
@@ -346,8 +345,8 @@ struct files_struct *dup_fd(struct files_struct *oldf, unsigned int max_fds, int new_fdt =alloc_fdtable (open_files);if (IS_ERR (new_fdt)) { - *errorp =PTR_ERR (new_fdt); - goto out_release; +kmem_cache_free (files_cachep, newf); +return ERR_CAST (new_fdt); }/*
@@ -388,11 +387,6 @@ struct files_struct *dup_fd(struct files_struct *oldf, unsigned int max_fds, intrcu_assign_pointer (newf->fdt, new_fdt);return newf; - -out_release: -kmem_cache_free (files_cachep, newf); -out: -return NULL ; }static struct fdtable *close_files (struct files_struct * files)
include/linux/fdtable.h +1 -1
@@ -102,7 +102,7 @@ struct task_struct;void put_files_struct (struct files_struct *fs);int unshare_files (void ); -struct files_struct *dup_fd (struct files_struct *,unsigned ,int *) __latent_entropy; +struct files_struct *dup_fd (struct files_struct *,unsigned ) __latent_entropy;void do_close_on_exec (struct files_struct *); int iterate_fd(struct files_struct *, unsigned, int (*)(const void *, struct file *, unsigned),
kernel/fork.c +12 -14
@@ -1754,33 +1754,30 @@ static int copy_files(unsigned long clone_flags, struct task_struct *tsk, int no_files) {struct files_struct *oldf, *newf; -int error =0 ;/* * A background process may not have any files ... */ oldf = current->files;if (!oldf) - goto out; +return 0 ;if (no_files) { tsk->files =NULL ; - goto out; +return 0 ; }if (clone_flags &CLONE_FILES ) {atomic_inc (&oldf->count); - goto out; +return 0 ; } - newf =dup_fd (oldf,NR_OPEN_MAX , &error); -if (!newf) - goto out; + newf =dup_fd (oldf,NR_OPEN_MAX ); +if (IS_ERR (newf)) +return PTR_ERR (newf); tsk->files = newf; - error =0 ; -out: -return error; +return 0 ; }static int copy_sighand (unsigned long clone_flags,struct task_struct *tsk)
@@ -3236,13 +3233,14 @@ int unshare_fd(unsigned long unshare_flags, unsigned int max_fds, struct files_struct **new_fdp) {struct files_struct *fd = current->files; -int error =0 ;if ((unshare_flags &CLONE_FILES ) && (fd &&atomic_read (&fd->count) >1 )) { - *new_fdp =dup_fd (fd, max_fds, &error); -if (!*new_fdp) -return error; + *new_fdp =dup_fd (fd, max_fds); +if (IS_ERR (*new_fdp)) { + *new_fdp =NULL ; +return PTR_ERR (new_fdp); + } }return 0 ;
Signature · 1 line
2.39.2
PATCH 1/1 close_files(): reimplement based on do_close_on_exec()
In reply to Al ViroWhile here take more advantage of the fact nobody should be messing with the table anymore and don't clear the fd slot.
---
how about this instead, I think it's a nicer clean up.
It's literally do_close_on_exec except locking and put fd are deleted.
boots & does not blow up, but admittedly I did not bother with ltp or any serious testing
fs/file.c | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-)
fs/file.c +21 -16
@@ -389,33 +389,38 @@ struct files_struct *dup_fd(struct files_struct *oldf, unsigned int max_fds)return newf; } -static struct fdtable *close_files (struct files_struct * files) +static struct fdtable *close_files (struct files_struct *files) {/* * It is safe to dereference the fd table without RCU or * ->file_lock because this is the last reference to the * files structure. +* +* For the same reason we can skip locking. */ struct fdtable *fdt =rcu_dereference_raw (files->fdt); -unsigned int i, j =0 ; +unsigned i; -for (;;) { +for (i =0 ; ; i++) {unsigned long set; - i = j *BITS_PER_LONG ; -if (i >= fdt->max_fds) +unsigned fd = i *BITS_PER_LONG ; + fdt =files_fdtable (files); +if (fd >= fdt->max_fds)break ; - set = fdt->open_fds[j++]; -while (set) { -if (set &1 ) { -struct file * file =xchg (&fdt->fd[i],NULL ); -if (file) { -filp_close (file, files); -cond_resched (); - } - } - i++; - set >>=1 ; + set = fdt->open_fds[i]; +if (!set) +continue ; +for ( ; set ; fd++, set >>=1 ) { +struct file *file; +if (!(set &1 )) +continue ; + file = fdt->fd[fd]; +if (!file) +continue ; +filp_close (file, files); +cond_resched (); } + }return fdt;
Signature · 1 line
2.43.0
Re: [PATCH 01/11] get rid of ...lookup...fdget_rcu() family
In reply to Al ViroQuoted 14 lines · Al Viro
Once upon a time, predecessors of those used to do file lookup without bumping a refcount, provided that caller held
rcu_read_lock()across the lookup and whatever it wanted to read from the struct file found. When struct file allocation switched toSLAB_TYPESAFE_BY_RCU, that stopped being feasible and these primitives started to bump the file refcount for lookup result, requiring the caller to callfput()afterwards.But that turned them pointless - e.g.
rcu_read_lock(); file = lookup_fdget_rcu(fd); rcu_read_unlock(); is equivalent to file = fget_raw(fd);
Quoted 8 lines
and all callers of
lookup_fdget_rcu()are of that form. Similarly,task_lookup_fdget_rcu()calls can be replaced with callingfget_task().task_lookup_next_fdget_rcu()doesn't have direct counterparts, but its callers would be happier if we replaced it with an analogue that deals with RCU internally.Signed-off-by Al Viro <[email protected]>---
Re: [PATCH 02/11] remove pointless includes of <linux/fdtable.h>
In reply to Al ViroQuoted 5 lines · Al Viro
some of those used to be needed, some had been cargo-culted for no reason...
Signed-off-by Al Viro <[email protected]>---
Re: [PATCH 06/11] sane_fdtable_size(): don't bother looking at descriptors we are not going to copy
In reply to Al ViroQuoted 6 lines · Al Viro
when given a
max_fdsargument lower than that current size (that can happen when called fromclose_range(..., CLOSE_RANGE_UNSHARE)), we can ignore all descriptors >=max_fds.Signed-off-by Al Viro <[email protected]>---
Re: [PATCH 11/11] dup_fd(): change calling conventions
In reply to Al ViroQuoted 4 lines · Al Viro
return
ERR_PTR()on failure, get rid of errorpSigned-off-by Al Viro <[email protected]>---
Re: [PATCH 10/11] alloc_fdtable(): change calling conventions.
In reply to Al ViroQuoted 21 lines · Al Viro
First of all, tell it how many slots do we want, not which slot is wanted. It makes one caller (
dup_fd()) more straightforward and doesn't harm another (expand_fdtable()).Furthermore, make it return
ERR_PTR()on failure rather than returning NULL. Simplifies the callers.Simplify the size calculation, while we are at it - note that we always have
slots_wantedgreater thanBITS_PER_LONG. What the rules boil down to is
* use the smallest power of two large enough to give us
that many slots
* on 32bit skip 64 and 128 - the minimal capacity we want
there is 256 slots (i.e. 1Kb fd array).
* on 64bit don't skip anything, the minimal capacity is
128 - and we'll never be asked for 64 or less. 128 slots means 1Kb fd array, again.
* on 128bit, if that ever happens, don't skip anything -
we'll never be asked for 128 or less, so the fd array allocation will be at least 2Kb.
Quoted 2 lines
Signed-off-by Al Viro <[email protected]>---
I'd copy the rules from the commit message into the function comment (Maybe not the 128bits part).
Al Viro
maintainerRe: [PATCH] close_files(): reimplement based on do_close_on_exec()
In reply to Mateusz GuzikQuoted 9 lines · Mateusz Guzik
While here take more advantage of the fact nobody should be messing with the table anymore and don't clear the fd slot.
Signed-off-by Mateusz Guzik <[email protected]>---
how about this instead, I think it's a nicer clean up.
It's literally
do_close_on_execexcept locking and put fd are deleted.
TBH, I don't see much benefit that way - if anything, you are doing a bunch of extra READ_ONCE() of the same thing (files->fdt), for no visible reason...
Re: [PATCH] close_files(): reimplement based on do_close_on_exec()
Quoted 8 lines · Al Viro
Quoted 6 lines · Mateusz Guzik
While here take more advantage of the fact nobody should be messing with the table anymore and don't clear the fd slot.
Signed-off-by Mateusz Guzik <[email protected]>---
Quoted 7 lines
Quoted 3 lines
how about this instead, I think it's a nicer clean up.
It's literally
do_close_on_execexcept locking and put fd are deleted.TBH, I don't see much benefit that way - if anything, you are doing a bunch of extra
READ_ONCE()of the same thing (files->fdt), for no visible reason...
I claim the stock code avoidably implements traversal differently from do_close_on_exec.
The fdt reload can be trivially lifted out of the loop, does not affect what I was going for.
But now that you mention this can also be done in the do_close_on_exec case -- the thread calling it is supposed to be the only consumer, so fdt can't change.
that's my $0,03 here, I'm not going to further argue about it
Signature · 1 line
Mateusz Guzik <mjguzik gmail.com>