fdtable: Make fdarray and fdsets equal in size.
PATCH 2/5 fdtable: Make fdarray and fdsets equal in size.
Currently, each fdtable supports three dynamically-sized arrays of data: the fdarray and two fdsets. The code allows the number of fds supported by the fdarray (fdtable->max_fds) to differ from the number of fds supported by each of the fdsets (fdtable->max_fdset). In practice, it is wasteful for these two sizes to differ: whenever we hit a limit on the smaller-capacity structure, we will reallocate the entire fdtable and all the dynamic arrays within it, so any delta in the memory used by the larger-capacity structure will never be touched at all. Rather than hogging this excess, we shouldn't even allocate it in the first place, and keep the capacities of the fdarray and the fdsets equal. This patch removes fdtable->max_fdset. As an added bonus, most of the supporting code becomes simpler.
arch/alpha/kernel/osf_sys.c +3 -3
@@ -979,7 +979,7 @@ osf_select(int n, fd_set __user *inp, fdlong timeout;int ret = -EINVAL ;struct fdtable *fdt; -int max_fdset; +int max_fds; timeout =MAX_SCHEDULE_TIMEOUT ;if (tvp) {
@@ -1003,9 +1003,9 @@ osf_select(int n, fd_set __user *inp, fdrcu_read_lock (); fdt =files_fdtable (current->files); - max_fdset = fdt->max_fdset; + max_fds = fdt->max_fds;rcu_read_unlock (); -if (n <0 || n > max_fdset) +if (n <0 || n > max_fds) goto out_nofds;/*
arch/mips/kernel/kspd.c +1 -1
@@ -301,7 +301,7 @@ static void sp_cleanup(void)for (;;) {unsigned long set; i = j * __NFDBITS; -if (i >= fdt->max_fdset || i >= fdt->max_fds) +if (i >= fdt->max_fds)break ; set = fdt->open_fds->fds_bits[j++];while (set) {
fs/compat.c +5 -5
@@ -1674,19 +1674,19 @@ int compat_core_sys_select(int n, compat {fd_set_bits fds;char *bits; -int size, max_fdset, ret = -EINVAL ; +int size, max_fds, ret = -EINVAL ;struct fdtable *fdt;if (n <0 ) goto out_nofds; -/* max_fdset can increase, so grab it once to avoid race */ +/* max_fds can increase, so grab it once to avoid race */ rcu_read_lock (); fdt =files_fdtable (current->files); - max_fdset = fdt->max_fdset; + max_fds = fdt->max_fds;rcu_read_unlock (); -if (n > max_fdset) - n = max_fdset; +if (n > max_fds) + n = max_fds;/* * We need 6 bitmaps (in/out/ex for both incoming and outgoing),
fs/exec.c +1 -1
@@ -782,7 +782,7 @@ static void flush_old_files(struct files j++; i = j * __NFDBITS; fdt =files_fdtable (files); -if (i >= fdt->max_fds || i >= fdt->max_fdset) +if (i >= fdt->max_fds)break ; set = fdt->close_on_exec->fds_bits[j];if (!set)
fs/fcntl.c +2 -3
@@ -77,10 +77,9 @@ repeat: start = files->next_fd; newfd = start; -if (start < fdt->max_fdset) { +if (start < fdt->max_fds) newfd =find_next_zero_bit (fdt->open_fds->fds_bits, - fdt->max_fdset, start); - } + fdt->max_fds, start); error = -EMFILE ;if (newfd >= current->signal->rlim[RLIMIT_NOFILE ].rlim_cur)
fs/file.c +22 -34
138 lines not on this page · raw message
fs/open.c +1 -2
@@ -864,8 +864,7 @@ int get_unused_fd(void) repeat: fdt =files_fdtable (files); - fd =find_next_zero_bit (fdt->open_fds->fds_bits, - fdt->max_fdset, + fd =find_next_zero_bit (fdt->open_fds->fds_bits, fdt->max_fds, files->next_fd);/*
fs/select.c +5 -5
@@ -311,7 +311,7 @@ static int core_sys_select(int n, fd_set {fd_set_bits fds;void *bits; -int ret, max_fdset; +int ret, max_fds;unsigned int size;struct fdtable *fdt;/* Allocate small arguments on the stack to save memory and be faster */
@@ -321,13 +321,13 @@ static int core_sys_select(int n, fd_setif (n <0 ) goto out_nofds; -/* max_fdset can increase, so grab it once to avoid race */ +/* max_fds can increase, so grab it once to avoid race */ rcu_read_lock (); fdt =files_fdtable (current->files); - max_fdset = fdt->max_fdset; + max_fds = fdt->max_fds;rcu_read_unlock (); -if (n > max_fdset) - n = max_fdset; +if (n > max_fds) + n = max_fds;/* * We need 6 bitmaps (in/out/ex for both incoming and outgoing),
include/linux/file.h +0 -6
@@ -26,14 +26,8 @@ struct embedded_fd_set {unsigned long fds_bits[1 ]; }; -/* -* More than this number of fds: we use a separately allocated fd_set -*/ -#define EMBEDDED_FD_SET_SIZE (BITS_PER_BYTE * sizeof(struct embedded_fd_set)) -struct fdtable {unsigned int max_fds; -int max_fdset;struct file ** fd;/* current fd array */ fd_set *close_on_exec;fd_set *open_fds;
include/linux/init_task.h +0 -1
@@ -11,7 +11,6 @@#define INIT_FDTABLE \ { \ .max_fds = NR_OPEN_DEFAULT, \ - .max_fdset = EMBEDDED_FD_SET_SIZE, \ .fd = &init_files.fd_array[0], \ .close_on_exec = (fd_set *)&init_files.close_on_exec_init, \ .open_fds = (fd_set *)&init_files.open_fds_init, \
kernel/exit.c +1 -1
@@ -428,7 +428,7 @@ static void close_files(struct files_strfor (;;) {unsigned long set; i = j * __NFDBITS; -if (i >= fdt->max_fdset || i >= fdt->max_fds) +if (i >= fdt->max_fds)break ; set = fdt->open_fds->fds_bits[j++];while (set) {
kernel/fork.c +6 -18
@@ -596,7 +596,7 @@ static inline int copy_fs(unsigned longstatic int count_open_files (struct fdtable *fdt) { -int size = fdt->max_fdset; +int size = fdt->max_fds;int i;/* Find the last open fd */
@@ -623,7 +623,6 @@ static struct files_struct *alloc_files( newf->next_fd =0 ; fdt = &newf->fdtab; fdt->max_fds =NR_OPEN_DEFAULT ; - fdt->max_fdset =EMBEDDED_FD_SET_SIZE ; fdt->close_on_exec = (fd_set *)&newf->close_on_exec_init; fdt->open_fds = (fd_set *)&newf->open_fds_init; fdt->fd = &newf->fd_array[0 ];
@@ -644,7 +643,7 @@ static struct files_struct *dup_fd(struc {struct files_struct *newf;struct file **old_fds, **new_fds; -int open_files, size, i, expand; +int open_files, size, i;struct fdtable *old_fdt, *new_fdt; *errorp = -ENOMEM ;
@@ -655,25 +654,14 @@ static struct files_struct *dup_fd(strucspin_lock (&oldf->file_lock); old_fdt =files_fdtable (oldf); new_fdt =files_fdtable (newf); - size = old_fdt->max_fdset; open_files =count_open_files (old_fdt); - expand =0 ;/* -* Check whether we need to allocate a larger fd array or fd set. -* Note: we're not a clone task, so the open count won't change. +* Check whether we need to allocate a larger fd array and fd set. +* Note: we're not a clone task, so the open count won't change. */ -if (open_files > new_fdt->max_fdset) { - new_fdt->max_fdset =0 ; - expand =1 ; - }if (open_files > new_fdt->max_fds) { new_fdt->max_fds =0 ; - expand =1 ; - } - -/* if the old fdset gets grown now, we'll only copy up to "size" fds */ -if (expand) {spin_unlock (&oldf->file_lock);spin_lock (&newf->file_lock); *errorp =expand_files (newf, open_files-1 );
@@ -719,8 +707,8 @@ static struct files_struct *dup_fd(struc/* This is long word aligned thus could use a optimized version */ memset (new_fds,0 , size); -if (new_fdt->max_fdset > open_files) { -int left = (new_fdt->max_fdset-open_files)/8 ; +if (new_fdt->max_fds > open_files) { +int left = (new_fdt->max_fds-open_files)/8 ;int start = open_files / (8 *sizeof (unsigned long ));memset (&new_fdt->open_fds->fds_bits[start],0 , left);
security/selinux/hooks.c +1 -1
@@ -1728,7 +1728,7 @@ static inline void flush_unauthorized_fi j++; i = j * __NFDBITS; fdt =files_fdtable (files); -if (i >= fdt->max_fds || i >= fdt->max_fdset) +if (i >= fdt->max_fds)break ; set = fdt->open_fds->fds_bits[j];if (!set)
Quoted 12 lines · Vadim Lobanov
Currently, each fdtable supports three dynamically-sized arrays of data: the fdarray and two fdsets. The code allows the number of fds supported by the fdarray (
fdtable->max_fds) to differ from the number of fds supported by each of the fdsets (fdtable->max_fdset). In practice, it is wasteful for these two sizes to differ: whenever we hit a limit on the smaller-capacity structure, we will reallocate the entire fdtable and all the dynamic arrays within it, so any delta in the memory used by the larger-capacity structure will never be touched at all. Rather than hogging this excess, we shouldn't even allocate it in the first place, and keep the capacities of the fdarray and the fdsets equal. This patch removesfdtable->max_fdset. As an added bonus, most of the supporting code becomes simpler.
Quoted 1 line
Signed-off-by Vadim Lobanov <[email protected]>
I agree with this patch in principle that it saves space and makes the lockfree fdtable code simpler. However, it would be nice if Viro or Christoph has a look at this and comment on why we always had two different sizes.
Thanks
Dipankar
Quoted 16 lines · Vadim Lobanov
fs/file.c +0
-12- nfds =max_t (int,8 *L1_CACHE_BYTES ,roundup_pow_of_two (nr +1 )); -if (nfds >NR_OPEN ) - nfds =NR_OPEN ; - - new_openset =alloc_fdset (nfds); - new_execset =alloc_fdset (nfds); -if (!new_openset || !new_execset) - goto out; - fdt->open_fds = new_openset; - fdt->close_on_exec = new_execset; - fdt->max_fdset = nfds; - nfds =NR_OPEN_DEFAULT ;/* * Expand to the max in easy steps, and keep expanding it until
fs/file.c static struct fdtable *alloc_fdtable(int · line 256
Quoted 3 lines
fs/file.c +0
-0@@ -271,15 +254,21 @@ static struct fdtable *alloc_fdtable(int nfds =NR_OPEN ; } }while (nfds <= nr);
If I understand well, we may allocate very small fdset, while previous minimum size was L1_CACHE_BYTES bytes. (512 bits for a 64 bytes cache line)
If you check commit 0c9e63fd38a2fb2181668a0cdd622a3c23cfd567,
(http://www.kernel.org/git/?p=linux/kernel/git/torvalds/linux-2.6.git;a=commit;h=0c9e63fd38a2fb2181668a0cdd622a3c23cfd567 ) you'll find this comment of mine :
3) Reduce size of allocated fdset. Currently two full pages are allocated, that is 32768 bits on x86 for example, and way too much. The minimum is now L1_CACHE_BYTES.
This minimum is mandatory to be sure two tasks wont share the same cache line to store their fdset (and possibly do lot of cache line ping pongs)
Eric
Quoted 29 lines · Eric Dumazet
Quoted 16 lines · Vadim Lobanov
fs/file.c +0
-12- nfds =max_t (int,8 *L1_CACHE_BYTES ,roundup_pow_of_two (nr +1 )); -if (nfds >NR_OPEN ) - nfds =NR_OPEN ; - - new_openset =alloc_fdset (nfds); - new_execset =alloc_fdset (nfds); -if (!new_openset || !new_execset) - goto out; - fdt->open_fds = new_openset; - fdt->close_on_exec = new_execset; - fdt->max_fdset = nfds; - nfds =NR_OPEN_DEFAULT ;/* * Expand to the max in easy steps, and keep expanding it until fs/file.c static struct fdtable *alloc_fdtable(int · line 256
Quoted 3 lines
fs/file.c +0
-0@@ -271,15 +254,21 @@ static struct fdtable *alloc_fdtable(int nfds =NR_OPEN ; } }while (nfds <= nr);If I understand well, we may allocate very small fdset, while previous minimum size was
L1_CACHE_BYTESbytes. (512 bits for a 64 bytes cache line)If you check commit
0c9e63fd38a2fb2181668a0cdd622a3c23cfd567,
(http://www.kernel.org/git/?p=linux/kernel/git/torvalds/linux-2.6.git;a=com
mit;h=0c9e63fd38a2fb2181668a0cdd622a3c23cfd567 ) you'll find this comment of mine :
Quoted 6 lines
3) Reduce size of allocated fdset. Currently two full pages are allocated, that is 32768 bits on x86 for example, and way too much. The minimum is now
L1_CACHE_BYTES.This minimum is mandatory to be sure two tasks wont share the same cache line to store their fdset (and possibly do lot of cache line ping pongs)
Good point. This should be easy to fix -- we can simply bump up the allocation 'nr' when calling alloc_fdmem() for the fdset. Since we won't cross a page size, the rest of the code will still stay the same.
I'll code this up once I track down why other folks are experiencing oopses.
Quoted 1 line
Eric
-- Vadim Lobanov