fdmap v2 - implements sequential and non-sequential fd allocations using fdmap
PATCH 2/8 fdmap v2 - implements sequential and non-sequential fd allocations using fdmap
This patch makes the kernel use the fdmap allocator for sequential (legacy) and non-sequential file descriptor allocations. It consolidates all scattered fdtable internal members accesses into fdmap.{c,h}.
Semantics of sequential file descriptor allocations remains unchanged. They are still performed in the same way the current fdtable bitmap does. As far as non-sequential file descriptor allocations goes, we do not allow F_DUPFD to allocate a file descriptor inside the non-sequential area. The sys_dup2() system call has been changed to allow over-writing an already allocated file descriptor inside the non-sequential area. It cannot be used (sys_dup2()) to allocate a new file descriptor in the non-sequential area. The base for the non-sequential area is randomly chosen. On top of that, being FIFO, it's not even easy to predict after the application run for a while. A simple test program for sys_nonseqfd(), sys_socket2() and O_NONSEQFD is availble here (verified and tested on a P4 HT and a dual Opteron):
http://www.xmailserver.org/extfd-test.c
- Davide
include/linux/file.h +81 -41
@@ -11,52 +11,53 @@#include <linux/spinlock.h> #include <linux/rcupdate.h> #include <linux/types.h> +#include <linux/fdmap.h> /* -* The default fd array needs to be at least BITS_PER_LONG, -* as this is the granularity returned by copy_fdset(). +* Initial size for the non sequential file descriptor arena */ -#define NR_OPEN_DEFAULT BITS_PER_LONG +#define FDMAP_NONSEQ_SIZE 64U/* -* The embedded_fd_set is a small fd_set, -* suitable for most tasks (which open <= BITS_PER_LONG files) +* Base for non sequential file descriptors */ -struct embedded_fd_set { -unsigned long fds_bits[1 ]; -}; +#define FDMAP_NONSEQ_BASE (1U << 28) -struct fdtable { -unsigned int max_fds; -struct file ** fd;/* current fd array */ -fd_set *close_on_exec; -fd_set *open_fds; -struct rcu_head rcu; -struct fdtable *next; -}; +#define FDMAP_RANDOM_BITS 20 + +/* +* The default fd array needs to be at least BITS_PER_LONG, +* as this is the granularity returned by copy_fdset(). +*/ +#define NR_OPEN_DEFAULT BITS_PER_LONG/* * Open file table structure */ struct files_struct { -/* -* read mostly part -*/ +/* +* read mostly part +*/ atomic_t count; -struct fdtable *fdt; -struct fdtable fdtab; -/* -* written part on a separate cache line in SMP -*/ +/* +* Used for legacy (sequential) file descriptor allocations. +*/ +struct fd_map *lfmap; +/* +* Used for non-sequential file descriptor allocations. +*/ +struct fd_map *fmap; + +struct fd_map lfmap_embed; +/* +* written part on a separate cache line in SMP +*/ spinlock_t file_lock____cacheline_aligned_in_smp ; -int next_fd; -struct embedded_fd_set close_on_exec_init; -struct embedded_fd_set open_fds_init; -struct file * fd_array[NR_OPEN_DEFAULT ]; +int fd_count; +struct list_head fdmap_slots[NR_OPEN_DEFAULT ]; +unsigned long fdmap_map[FDMAP_BMP_LONGS (NR_OPEN_DEFAULT )]; }; -#define files_fdtable (files) (rcu_dereference((files)->fdt)) -extern struct kmem_cache *filp_cachep;extern void FASTCALL (__fput (struct file *));
@@ -73,25 +74,63 @@extern void FASTCALL (set_close_on_exec (unsigned int fd,int flag));extern void put_filp (struct file *);extern int get_unused_fd (void ); +extern void __put_unused_fd (unsigned int fd);extern void FASTCALL (put_unused_fd (unsigned int fd));struct kmem_cache ; -extern int expand_files (struct files_struct *,int nr); -extern void free_fdtable_rcu (struct rcu_head *rcu); -extern void __init files_defer_init (void ); - -static inline void free_fdtable (struct fdtable *fdt) +extern void init_files_struct (struct files_struct *newf); +extern struct fd_map *files_fdmap_alloc (struct files_struct *files, +struct fd_map **pfmap, +unsigned int size); +extern int __alloc_nonseq_fd (struct files_struct *files,unsigned long flags); +extern int alloc_nonseq_fd (struct files_struct *files,unsigned long flags); +extern unsigned int gen_nonseqfd_base (void ); + +/** +* nonseq_files_fdmap - Must be called with @files->file_lock held. It forces +* the creation the the non-sequential file descriptor map, +* if not present. +* +* @files: [in] Pointer to the struct files_struct from where the map has to +* be retrieved +* +*/ +static inline struct fd_map *nonseq_files_fdmap (struct files_struct *files) { -call_rcu (&fdt->rcu, free_fdtable_rcu); +struct fd_map *fmap = files->fmap; +if (unlikely (!fmap)) + fmap =files_fdmap_alloc (files, &files->fmap,0 ); +return fmap; } -static inline struct file *fcheck_files (struct files_struct *files,unsigned int fd) +/** +* files_free_fdmap - Frees the non-sequential file descriptor map, if it is +* not the files-embedded one (files->lfmap_embed) +* +* @files: [in] Pointer to the struct files_struct +* @fmap: [in] File descriptor map to be freed +* +*/ +static inline void files_free_fdmap (struct files_struct *files, +struct fd_map *fmap) { -struct file * file =NULL ; -struct fdtable *fdt =files_fdtable (files); +if (fmap != &files->lfmap_embed) +fdmap_free (fmap); +} -if (fd < fdt->max_fds) - file =rcu_dereference (fdt->fd[fd]); +static inline struct file *fcheck_files (struct files_struct *files, +unsigned int fd) +{ +struct file *file =NULL ; +struct fd_map *fmap; + fmap =rcu_dereference (files->lfmap); +if (fdmap_fdof (fmap, fd)) + file =fdmap_file_get (fmap, fd); +else { + fmap =rcu_dereference (files->fmap); +if (fmap &&fdmap_fdof (fmap, fd)) + file =fdmap_file_get (fmap, fd); + }return file; }
@@ -105,6 +144,7 @@struct task_struct ;struct files_struct *get_files_struct (struct task_struct *); +void free_files_struct (struct files_struct *files);void FASTCALL (put_files_struct (struct files_struct *fs));void reset_files_struct (struct task_struct *,struct files_struct *);
fs/fcntl.c +72 -99
217 lines not on this page · raw message
fs/exec.c +21 -20
57 lines not on this page · raw message
kernel/exit.c +30 -44
90 lines not on this page · raw message
fs/open.c +106 -57
219 lines not on this page · raw message
kernel/fork.c +57 -94
186 lines not on this page · raw message
include/linux/init_task.h +4 -21
@@ -9,27 +9,10 @@#include <linux/ipc.h> #include <linux/pid_namespace.h> -#define INIT_FDTABLE \ -{ \ - .max_fds = NR_OPEN_DEFAULT, \ - .fd = &init_files.fd_array[0], \ - .close_on_exec = (fd_set *)&init_files.close_on_exec_init, \ - .open_fds = (fd_set *)&init_files.open_fds_init, \ - .rcu = RCU_HEAD_INIT, \ - .next = NULL, \ -} - -#define INIT_FILES \ -{ \ - .count = ATOMIC_INIT(1), \ - .fdt = &init_files.fdtab, \ - .fdtab = INIT_FDTABLE, \ - .file_lock = __SPIN_LOCK_UNLOCKED(init_task.file_lock), \ - .next_fd = 0, \ - .close_on_exec_init = { { 0, } }, \ - .open_fds_init = { { 0, } }, \ - .fd_array = { NULL, } \ -} +/* +* We do the real "init_files" initialization inside fs/file_table.c:files_init() +*/ +#define INIT_FILES { }#define INIT_KIOCTX (name, which_mm) \ { \
kernel/kmod.c +6 -5
@@ -146,17 +146,18 @@/* Install input pipe when needed */ if (sub_info->stdin) { +int fd;struct files_struct *f = current->files; -struct fdtable *fdt;/* no races because files should be private here */ sys_close (0 ); -fd_install (0 , sub_info->stdin);spin_lock (&f->file_lock); - fdt =files_fdtable (f); -FD_SET (0 , fdt->open_fds); -FD_CLR (0 , fdt->close_on_exec); + fd =fdmap_newfd (f->lfmap,0 ,0 ); +BUG_ON (fd <0 ); + f->fd_count++;spin_unlock (&f->file_lock); +fd_install (0 , sub_info->stdin); +/* and disallow core files too */ current->signal->rlim[RLIMIT_CORE ] = (struct rlimit ){0 ,0 }; }
fs/proc/base.c +21 -10
@@ -1384,10 +1384,10 @@struct dentry *dentry = filp->f_path.dentry;struct inode *inode = dentry->d_inode;struct task_struct *p =get_proc_task (inode); -unsigned int fd, tid, ino; +unsigned int fd, tid, ino, topfd;int retval;struct files_struct * files; -struct fdtable *fdt; +struct fd_map *lfmap, *fmap; retval = -ENOENT ;if (!p)
@@ -1411,10 +1411,15 @@if (!files) goto out;rcu_read_lock (); - fdt =files_fdtable (files); -for (fd = filp->f_pos-2 ; - fd < fdt->max_fds; - fd++, filp->f_pos++) { + lfmap =rcu_dereference (files->lfmap); + fmap =rcu_dereference (files->fmap); + fd = filp->f_pos -2 ; +if (fd <fdmap_topfd (lfmap) || !fmap) + topfd =fdmap_topfd (lfmap); +else + topfd =fdmap_topfd (fmap); +rescan: +for (; fd < topfd; fd++, filp->f_pos++) {char name[PROC_NUMBUF ];int len;
@@ -1425,13 +1430,19 @@ len =snprintf (name,sizeof (name),"%d" , fd);if (proc_fill_cache (filp, dirent, filldir, name, len, instantiate, - p, &fd) <0 ) { -rcu_read_lock (); -break ; - } + p, &fd) <0 ) + goto out_put_files;rcu_read_lock (); } + fmap =rcu_dereference (files->fmap); +if (fmap && fd <fdmap_basefd (fmap)) { + fd =fdmap_basefd (fmap); + filp->f_pos = fd +2 ; + topfd =fdmap_topfd (fmap); + goto rescan; + }rcu_read_unlock (); +out_put_files:put_files_struct (files); } out:
fs/file.c +47 -219
286 lines not on this page · raw message
fs/file_table.c +2 -1
@@ -298,6 +298,7 @@ files_stat.max_files = n;if (files_stat.max_files <NR_FILE ) files_stat.max_files =NR_FILE ; -files_defer_init (); +fdmap_module_init (); +init_files_struct (init_task.files);percpu_counter_init (&nr_files,0 ); }
fs/proc/array.c +3 -3
@@ -160,7 +160,7 @@ {struct group_info *group_info;int g; -struct fdtable *fdt =NULL ; +struct fd_map *fmap =NULL ;rcu_read_lock (); buffer += sprintf(buffer,
@@ -182,11 +182,11 @@task_lock (p);if (p->files) - fdt =files_fdtable (p->files); + fmap =rcu_dereference (p->files->lfmap); buffer +=sprintf (buffer,"FDSize:\t%d\n" "Groups:\t" , - fdt ? fdt->max_fds :0 ); + fmap ?fdmap_topfd (fmap):0 );rcu_read_unlock (); group_info = p->group_info;
security/selinux/hooks.c +55 -53
125 lines not on this page · raw message
fs/select.c +4 -7
@@ -139,16 +139,15 @@static int max_select_fd (unsigned long n,fd_set_bits *fds) { -unsigned long *open_fds; +const unsigned long *open_fds;unsigned long set;int max; -struct fdtable *fdt;/* handle last in-complete long-word first */ set = ~(~0UL << (n & (__NFDBITS-1 ))); n /= __NFDBITS; - fdt =files_fdtable (current->files); - open_fds = fdt->open_fds->fds_bits+n; + open_fds =fdmap_get_allocmap (current->files->lfmap); + open_fds += n; max =0 ;if (set) { set &=BITS (fds, n);
@@ -312,7 +311,6 @@void *bits;int ret, max_fds;unsigned int size; -struct fdtable *fdt;/* Allocate small arguments on the stack to save memory and be faster */ long stack_fds[SELECT_STACK_ALLOC /sizeof (long )];
@@ -322,8 +320,7 @@/* max_fds can increase, so grab it once to avoid race */ rcu_read_lock (); - fdt =files_fdtable (current->files); - max_fds = fdt->max_fds; + max_fds =fdmap_topfd (current->files->lfmap);rcu_read_unlock ();if (n > max_fds) n = max_fds;
fs/compat.c +1 -3
@@ -1546,7 +1546,6 @@fd_set_bits fds;void *bits;int size, max_fds, ret = -EINVAL ; -struct fdtable *fdt;long stack_fds[SELECT_STACK_ALLOC /sizeof (long )];if (n <0 )
@@ -1554,8 +1553,7 @@/* max_fds can increase, so grab it once to avoid race */ rcu_read_lock (); - fdt =files_fdtable (current->files); - max_fds = fdt->max_fds; + max_fds =fdmap_topfd (current->files->lfmap);rcu_read_unlock ();if (n > max_fds) n = max_fds;
drivers/char/tty_io.c +27 -17
67 lines not on this page · raw message
arch/alpha/kernel/osf_sys.c +1 -3
@@ -986,7 +986,6 @@size_t size;long timeout;int ret = -EINVAL ; -struct fdtable *fdt;int max_fds; timeout =MAX_SCHEDULE_TIMEOUT ;
@@ -1010,8 +1009,7 @@ }rcu_read_lock (); - fdt =files_fdtable (current->files); - max_fds = fdt->max_fds; + max_fds =fdmap_topfd (current->files->lfmap);rcu_read_unlock ();if (n <0 || n > max_fds) goto out_nofds;
arch/ia64/kernel/perfmon.c +1 -12
@@ -2259,20 +2259,9 @@static void pfm_free_fd (int fd,struct file *file) { -struct files_struct *files = current->files; -struct fdtable *fdt; - -/* -* there ie no fd_uninstall(), so we do it here -*/ -spin_lock (&files->file_lock); - fdt =files_fdtable (files); -rcu_assign_pointer (fdt->fd[fd],NULL ); -spin_unlock (&files->file_lock); - +put_unused_fd (current->files, fd);if (file)put_filp (file); -put_unused_fd (fd); } static int
arch/mips/kernel/kspd.c +9 -25
@@ -294,35 +294,19 @@printk ("KSPD: sp_work_handle_request failed to send to SP\n" ); } +static int sp_files_fdmap_close (void *priv,struct file *file,int fd) +{ +filp_close (file, (struct files_struct *) priv); +return 0 ; +} +static void sp_cleanup (void ) {struct files_struct *files = current->files; -int i, j; -struct fdtable *fdt; - j =0 ; - -/* -* It is safe to dereference the fd table without RCU or -* ->file_lock -*/ - fdt =files_fdtable (files); -for (;;) { -unsigned long set; - i = j * __NFDBITS; -if (i >= fdt->max_fds) -break ; - set = fdt->open_fds->fds_bits[j++]; -while (set) { -if (set &1 ) { -struct file * file =xchg (&fdt->fd[i],NULL ); -if (file) -filp_close (file, files); - } - i++; - set >>=1 ; - } - } +fdmap_for_each_file (files->lfmap,1 , sp_files_fdmap_close, files); +if (files->fmap) +fdmap_for_each_file (files->fmap,1 , sp_files_fdmap_close, files); }static int channel_open =0 ;
arch/powerpc/platforms/cell/spufs/coredump.c +20 -14
@@ -133,6 +133,20 @@return size; } +static int spufs_fdmap_add_one_context (void *priv,struct file *file,int fd) +{ +int *size = priv; + +if (file->f_op == &spufs_context_fops) { +int rval =spufs_add_one_context (file, fd); +if (rval <0 ) +return 1 ; + *size += rval; + } + +return 0 ; +} +/* * The additional architecture-specific notes for Cell are various * context files in the spu context.
@@ -144,21 +158,13 @@*/ static int spufs_arch_notes_size (void ) { -struct fdtable *fdt =files_fdtable (current->files); -int size =0 , fd; +struct files_struct *files = current->files; +int size =0 ; -for (fd =0 ; fd < fdt->max_fds; fd++) { -if (FD_ISSET (fd, fdt->open_fds)) { -struct file *file =fcheck (fd); - -if (file && file->f_op == &spufs_context_fops) { -int rval =spufs_add_one_context (file, fd); -if (rval <0 ) -break ; - size += rval; - } - } - } +fdmap_for_each_file (files->lfmap,0 , spufs_fdmap_add_one_context, &size); +if (files->fmap) +fdmap_for_each_file (files->fmap,0 , spufs_fdmap_add_one_context, + &size);return size; }
arch/sparc64/solaris/ioctl.c +5 -5
@@ -294,13 +294,13 @@static inline int solaris_sockmod (unsigned int fd,unsigned int cmd,u32 arg) {struct inode *ino; -struct fdtable *fdt; +struct file *file;/* I wonder which of these tests are superfluous... --patrik */ rcu_read_lock (); - fdt =files_fdtable (current->files); -if (! fdt->fd[fd] || - ! fdt->fd[fd]->f_path.dentry || - ! (ino = fdt->fd[fd]->f_path.dentry->d_inode) || + file =fcheck_files (current->files, fd); +if (! file || + ! file->f_path.dentry || + ! (ino = file->f_path.dentry->d_inode) || !S_ISSOCK (ino->i_mode)) {rcu_read_unlock ();return TBADF ;
arch/sparc64/solaris/timod.c +13 -21
@@ -143,11 +143,11 @@static void timod_wake_socket (unsigned int fd) {struct socket *sock; -struct fdtable *fdt; +struct file *file;SOLD ("wakeing socket" ); - fdt =files_fdtable (current->files); - sock =SOCKET_I (fdt->fd[fd]->f_path.dentry->d_inode); + file =fcheck_files (current->files, fd); + sock =SOCKET_I (file->f_path.dentry->d_inode);wake_up_interruptible (&sock->wait);read_lock (&sock->sk->sk_callback_lock);if (sock->fasync_list && !test_bit (SOCK_ASYNC_WAITDATA , &sock->flags))
@@ -159,11 +159,11 @@static void timod_queue (unsigned int fd,struct T_primsg *it) {struct sol_socket_struct *sock; -struct fdtable *fdt; +struct file *file;SOLD ("queuing primsg" ); - fdt =files_fdtable (current->files); - sock = (struct sol_socket_struct *)fdt->fd[fd]->private_data; + file =fcheck_files (current->files, fd); + sock = (struct sol_socket_struct *)file->private_data; it->next = sock->pfirst; sock->pfirst = it;if (!sock->plast)
@@ -175,11 +175,11 @@static void timod_queue_end (unsigned int fd,struct T_primsg *it) {struct sol_socket_struct *sock; -struct fdtable *fdt; +struct file *file;SOLD ("queuing primsg at end" ); - fdt =files_fdtable (current->files); - sock = (struct sol_socket_struct *)fdt->fd[fd]->private_data; + file =fcheck_files (current->files, fd); + sock = (struct sol_socket_struct *)file->private_data; it->next =NULL ;if (sock->plast) sock->plast->next = it;
@@ -350,7 +350,6 @@char *buf;struct file *filp;struct inode *ino; -struct fdtable *fdt;struct sol_socket_struct *sock;mm_segment_t old_fs =get_fs ();long args[6 ];
@@ -359,8 +358,7 @@int (*sys_sendto)(int ,void __user *,size_t ,unsigned ,struct sockaddr __user *,int ) = (int (*)(int ,void __user *,size_t ,unsigned ,struct sockaddr __user *,int ))SYS (sendto); - fdt =files_fdtable (current->files); - filp = fdt->fd[fd]; + filp =fcheck_files (current->files, fd); ino = filp->f_path.dentry->d_inode; sock = (struct sol_socket_struct *)filp->private_data;SOLD ("entry" );
@@ -629,7 +627,6 @@int oldflags;struct file *filp;struct inode *ino; -struct fdtable *fdt;struct sol_socket_struct *sock;struct T_unitdata_ind udi;mm_segment_t old_fs =get_fs ();
@@ -642,8 +639,7 @@SOLD ("entry" );SOLDD (("%u %p %d %p %p %d %p %d\n" , fd, ctl_buf, ctl_maxlen, ctl_len, data_buf, data_maxlen, data_len, *flags_p)); - fdt =files_fdtable (current->files); - filp = fdt->fd[fd]; + filp =fcheck_files (current->files, fd); ino = filp->f_path.dentry->d_inode; sock = (struct sol_socket_struct *)filp->private_data;SOLDD (("%p %p\n" , sock->pfirst, sock->pfirst ? sock->pfirst->next :NULL ));
@@ -855,14 +851,12 @@int __user *flgptr;int flags;int error = -EBADF ; -struct fdtable *fdt;SOLD ("entry" );lock_kernel ();if (fd >=NR_OPEN ) goto out; - fdt =files_fdtable (current->files); - filp = fdt->fd[fd]; + filp =fcheck_files (current->files, fd);if (!filp) goto out; ino = filp->f_path.dentry->d_inode;
@@ -923,14 +917,12 @@struct strbuf ctl, dat;int flags = (int ) arg3;int error = -EBADF ; -struct fdtable *fdt;SOLD ("entry" );lock_kernel ();if (fd >=NR_OPEN ) goto out; - fdt =files_fdtable (current->files); - filp = fdt->fd[fd]; + filp =fcheck_files (current->files, fd);if (!filp) goto out; ino = filp->f_path.dentry->d_inode;