fs/proc: Move kfree outside pde_unload_lock
Subsystems (2)
PATCH 1/1 fs/proc: Move kfree outside pde_unload_lock
This moves a kfree outside a spinlock to help scaling on larger (512 core) systems.
I ran a simple test which just reads from /proc/cpuinfo. Lower is better, as you can see the worst case scenario is improved.
baseline moved kfree
tasks read-sec read-sec
1 0.0141 0.0141
2 0.0140 0.0140
4 0.0140 0.0141
8 0.0145 0.0145
16 0.0553 0.0548
32 0.1688 0.1622
64 0.5017 0.3856
128 1.7005 0.9710
256 5.2513 2.6519
512 8.0529 6.2976
--- fs/proc/inode.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)
fs/proc/inode.c +1 -1
@@ -403,9 +403,9 @@ static int proc_reg_release(struct inode *inode, struct file *file) release = pde->proc_fops->release;if (pdeo) {list_del (&pdeo->lh); -kfree (pdeo); }spin_unlock (&pde->pde_unload_lock); +kfree (pdeo);if (release) rv =release (inode, file);
Signature · 1 line
1.6.0.2
PATCH 1/1 fs/prof: Update comment on pde_unload_lock
The comment was updated to include the other structures held by the lock.
--- include/linux/proc_fs.h | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-)
include/linux/proc_fs.h +2 -1
@@ -73,7 +73,8 @@ struct proc_dir_entry {int pde_users;/* number of callers into module in progress */ struct completion *pde_unload_completion;struct list_head pde_openers;/* who did ->open, but not ->release */ -spinlock_t pde_unload_lock;/* proc_fops checks and pde_users bumps */ +spinlock_t pde_unload_lock;/* proc_fops checks, pde_users bumps */ +/* pde_openers, pde_unload_completion */ u8 namelen;char name[]; };
Signature · 1 line
1.6.0.2
Quoted 32 lines · Nathan Zimmer
This moves a kfree outside a spinlock to help scaling on larger (512 core) systems.
I ran a simple test which just reads from
/proc/cpuinfo. Lower is better, as you can see the worst case scenario is improved.baseline moved kfree
tasks read-sec read-sec
1 0.0141 0.0141
2 0.0140 0.0140
4 0.0140 0.0141
8 0.0145 0.0145
16 0.0553 0.0548
32 0.1688 0.1622
64 0.5017 0.3856
128 1.7005 0.9710
256 5.2513 2.6519
512 8.0529 6.2976Cc Alexander Viro <[email protected]>Cc David Woodhouse <[email protected]>Acked-by Alexey Dobriyan <[email protected]>Signed-off-by Nathan Zimmer <[email protected]>--- fs/proc/inode.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)fs/proc/inode.c +0
-0
fs/proc/inode.c static int proc_reg_release(struct inode *inode, struct file *file) · line 411
Quoted 10 lines
fs/proc/inode.c +1
-1@@ -403,9 +403,9 @@ static int proc_reg_release(struct inode *inode, struct file *file) release = pde->proc_fops->release;if (pdeo) {list_del (&pdeo->lh); -kfree (pdeo); }spin_unlock (&pde->pde_unload_lock); +kfree (pdeo);if (release) rv =release (inode, file);
Thats interesting, but if you really want this to fly, one RCU conversion would be much better ;)
pde_users would be an atomic_t and you would avoid the spinlock contention.
Quoted 6 lines · Eric Dumazet
Thats interesting, but if you really want this to fly, one RCU conversion would be much better ;)
pde_userswould be anatomic_tand you would avoid the spinlock contention.
Here is what I had in mind, I would be interested to know how it helps a 512 core machine ;)
fs/proc/generic.c | 66 ++++------ fs/proc/inode.c | 250 +++++++++++++++++++++++--------------- fs/proc/internal.h | 2 include/linux/proc_fs.h | 7 - 4 files changed, 190 insertions(+), 135 deletions(-)
fs/proc/generic.c +30 -36
@@ -21,7 +21,7 @@#include <linux/namei.h> #include <linux/bitops.h> #include <linux/spinlock.h> -#include <linux/completion.h> +#include <linux/sched.h> #include <asm/uaccess.h> #include "internal.h"
@@ -190,14 +190,16 @@ proc_file_read(struct file *file, char __user *buf, size_t nbytes, {struct proc_dir_entry *pde =PDE (file->f_path.dentry->d_inode);ssize_t rv = -EIO ; +const struct file_operations *fops; -spin_lock (&pde->pde_unload_lock); -if (!pde->proc_fops) { -spin_unlock (&pde->pde_unload_lock); +rcu_read_lock (); + fops =rcu_dereference (pde->proc_fops); +if (!fops) { +rcu_read_unlock ();return rv; } - pde->pde_users++; -spin_unlock (&pde->pde_unload_lock); +atomic_inc (&pde->pde_users); +rcu_read_unlock (); rv =__proc_file_read (file, buf, nbytes, ppos);
@@ -213,13 +215,16 @@ proc_file_write(struct file *file, const char __user *buffer,ssize_t rv = -EIO ;if (pde->write_proc) { -spin_lock (&pde->pde_unload_lock); -if (!pde->proc_fops) { -spin_unlock (&pde->pde_unload_lock); +const struct file_operations *fops; + +rcu_read_lock (); + fops =rcu_dereference (pde->proc_fops); +if (!fops) { +rcu_read_unlock ();return rv; } - pde->pde_users++; -spin_unlock (&pde->pde_unload_lock); +atomic_inc (&pde->pde_users); +rcu_read_unlock ();/* FIXME: does this routine need ppos? probably... */ rv = pde->write_proc (file, buffer, count, pde->data);
@@ -564,7 +569,7 @@ static int proc_register(struct proc_dir_entry * dir, struct proc_dir_entry * dpif (S_ISDIR (dp->mode)) {if (dp->proc_iops ==NULL ) { - dp->proc_fops = &proc_dir_operations; +RCU_INIT_POINTER (dp->proc_fops, &proc_dir_operations); dp->proc_iops = &proc_dir_inode_operations; } dir->nlink++;
@@ -573,7 +578,7 @@ static int proc_register(struct proc_dir_entry * dir, struct proc_dir_entry * dp dp->proc_iops = &proc_link_inode_operations; }else if (S_ISREG (dp->mode)) {if (dp->proc_fops ==NULL ) - dp->proc_fops = &proc_file_operations; +RCU_INIT_POINTER (dp->proc_fops, &proc_file_operations);if (dp->proc_iops ==NULL ) dp->proc_iops = &proc_file_inode_operations; }
@@ -625,11 +630,8 @@ static struct proc_dir_entry *__proc_create(struct proc_dir_entry **parent, ent->mode = mode; ent->nlink = nlink;atomic_set (&ent->count,1 ); - ent->pde_users =0 ; -spin_lock_init (&ent->pde_unload_lock); - ent->pde_unload_completion =NULL ; -INIT_LIST_HEAD (&ent->pde_openers); - out: +atomic_set (&ent->pde_users,0 ); +out:return ent; }
@@ -751,7 +753,7 @@ struct proc_dir_entry *proc_create_data(const char *name, umode_t mode, pde =__proc_create (&parent, name, mode, nlink);if (!pde) goto out; - pde->proc_fops = proc_fops; +rcu_assign_pointer (pde->proc_fops, proc_fops); pde->data = data;if (proc_register (parent, pde) <0 ) goto out_free;
@@ -787,6 +789,7 @@ void remove_proc_entry(const char *name, struct proc_dir_entry *parent)struct proc_dir_entry *de =NULL ;const char *fn = name;unsigned int len; +LIST_HEAD (purge_queue);spin_lock (&proc_subdir_lock);if (__xlate_proc_name (name, &parent, &fn) !=0 ) {
@@ -809,37 +812,28 @@ void remove_proc_entry(const char *name, struct proc_dir_entry *parent)return ; } -spin_lock (&de->pde_unload_lock);/* * Stop accepting new callers into module. If you're * dynamically allocating ->proc_fops, save a pointer somewhere. */ de->proc_fops =NULL ; +synchronize_rcu ();/* Wait until all existing callers into module are done. */ -if (de->pde_users >0 ) { -DECLARE_COMPLETION_ONSTACK (c); - -if (!de->pde_unload_completion) - de->pde_unload_completion = &c; - -spin_unlock (&de->pde_unload_lock); - -wait_for_completion (de->pde_unload_completion); - -spin_lock (&de->pde_unload_lock); +while (atomic_read (&de->pde_users)) { +set_current_state (TASK_UNINTERRUPTIBLE ); +schedule (); } + current->state =TASK_RUNNING ; +pde_openers_purge (de, &purge_queue); -while (!list_empty (&de->pde_openers)) { +while (!list_empty (&purge_queue)) {struct pde_opener *pdeo; - pdeo = list_first_entry(&de->pde_openers, struct pde_opener, lh); + pdeo = list_first_entry(&purge_queue, struct pde_opener, lh);list_del (&pdeo->lh); -spin_unlock (&de->pde_unload_lock); pdeo->release (pdeo->inode, pdeo->file);kfree (pdeo); -spin_lock (&de->pde_unload_lock); } -spin_unlock (&de->pde_unload_lock);if (S_ISDIR (de->mode)) parent->nlink--;
fs/proc/inode.c +156 -94
396 lines not on this page · raw message
fs/proc/internal.h +2 -0
@@ -97,12 +97,14 @@ int proc_readdir_de(struct proc_dir_entry *de, struct file *filp, void *dirent, filldir_t filldir);struct pde_opener { +struct proc_dir_entry *pde;struct inode *inode;struct file *file;int (*release)(struct inode *,struct file *);struct list_head lh; };void pde_users_dec (struct proc_dir_entry *pde); +void pde_openers_purge (struct proc_dir_entry *pde,struct list_head *queue);extern spinlock_t proc_subdir_lock;
include/linux/proc_fs.h +2 -5
@@ -64,16 +64,13 @@ struct proc_dir_entry {* If you're allocating ->proc_fops dynamically, save a pointer * somewhere. */ -const struct file_operations *proc_fops; +const struct file_operations __rcu *proc_fops;struct proc_dir_entry *next, *parent, *subdir;void *data;read_proc_t *read_proc;write_proc_t *write_proc;atomic_t count;/* use count */ -int pde_users;/* number of callers into module in progress */ -struct completion *pde_unload_completion; -struct list_head pde_openers;/* who did ->open, but not ->release */ -spinlock_t pde_unload_lock;/* proc_fops checks and pde_users bumps */ +atomic_t pde_users;/* number of callers into module in progress */ u8 namelen;char name[]; };
Quoted 9 lines · Eric Dumazet
Quoted 5 lines · Eric Dumazet
Thats interesting, but if you really want this to fly, one RCU conversion would be much better ;)
pde_userswould be anatomic_tand you would avoid the spinlock contention.Here is what I had in mind, I would be interested to know how it helps a 512 core machine ;)
Thanks, I knew if I just took my time and read the rcu documentation thoroughly that the answer would be forthcoming. ;)
Unfortunately I have to wait till tomorrow to get big box and test it.
Quoted 11 lines · Eric Dumazet
Quoted 6 lines · Eric Dumazet
Thats interesting, but if you really want this to fly, one RCU conversion would be much better ;)
pde_userswould be anatomic_tand you would avoid the spinlock contention.Here is what I had in mind, I would be interested to know how it helps a 512 core machine ;)
Here are the results and they look great.
cpuinfo baseline moved kfree Rcu
tasks read-sec read-sec read-sec
1 0.0141 0.0141 0.0141
2 0.0140 0.0140 0.0142
4 0.0140 0.0141 0.0141
8 0.0145 0.0145 0.0140
16 0.0553 0.0548 0.0168
32 0.1688 0.1622 0.0549
64 0.5017 0.3856 0.1690
128 1.7005 0.9710 0.5038
256 5.2513 2.6519 2.0804
512 8.0529 6.2976 3.0162
Quoted 15 lines · Nathan Zimmer
Quoted 11 lines · Eric Dumazet
Quoted 6 lines · Eric Dumazet
Thats interesting, but if you really want this to fly, one RCU conversion would be much better ;)
pde_userswould be anatomic_tand you would avoid the spinlock contention.Here is what I had in mind, I would be interested to know how it helps a 512 core machine ;)
Here are the results and they look great.
Quoted 15 lines
cpuinfo baseline moved kfree Rcu
tasks read-sec read-sec read-sec
1 0.0141 0.0141 0.0141
2 0.0140 0.0140 0.0142
4 0.0140 0.0141 0.0141
8 0.0145 0.0145 0.0140
16 0.0553 0.0548 0.0168
32 0.1688 0.1622 0.0549
64 0.5017 0.3856 0.1690
128 1.7005 0.9710 0.5038
256 5.2513 2.6519 2.0804
512 8.0529 6.2976 3.0162
Indeed...
Could you explicit the test you are actually doing ?
Thanks
Quoted 28 lines · Eric Dumazet
Quoted 12 lines · Nathan Zimmer
Quoted 9 lines · Eric Dumazet
Quoted 5 lines · Eric Dumazet
Thats interesting, but if you really want this to fly, one RCU conversion would be much better ;)
pde_userswould be anatomic_tand you would avoid the spinlock contention.Here is what I had in mind, I would be interested to know how it helps a 512 core machine ;)
Here are the results and they look great.
Quoted 15 lines
cpuinfo baseline moved kfree Rcu
tasks read-sec read-sec read-sec
1 0.0141 0.0141 0.0141
2 0.0140 0.0140 0.0142
4 0.0140 0.0141 0.0141
8 0.0145 0.0145 0.0140
16 0.0553 0.0548 0.0168
32 0.1688 0.1622 0.0549
64 0.5017 0.3856 0.1690
128 1.7005 0.9710 0.5038
256 5.2513 2.6519 2.0804
512 8.0529 6.2976 3.0162
Quoted 7 lines
Indeed...
Could you explicit the test you are actually doing ?
Thanks
It is a dead simple test.
The test starts by forking off X number of tasks
assigning each their own cpu.
Each task then allocs a bit of memory.
All tasks wait on a memory cell for the go order.
We measure the read time starting here.
Once the go order is given they all read a chunk of the selected proc file. I was using /proc/cpuinfo to test.
Once everyone has finished we take the end read time.
Quoted 38 lines · Nathan Zimmer
Quoted 28 lines · Eric Dumazet
Quoted 12 lines · Nathan Zimmer
Quoted 9 lines · Eric Dumazet
Quoted 5 lines · Eric Dumazet
Thats interesting, but if you really want this to fly, one RCU conversion would be much better ;)
pde_userswould be anatomic_tand you would avoid the spinlock contention.Here is what I had in mind, I would be interested to know how it helps a 512 core machine ;)
Here are the results and they look great.
Quoted 15 lines
cpuinfo baseline moved kfree Rcu
tasks read-sec read-sec read-sec
1 0.0141 0.0141 0.0141
2 0.0140 0.0140 0.0142
4 0.0140 0.0141 0.0141
8 0.0145 0.0145 0.0140
16 0.0553 0.0548 0.0168
32 0.1688 0.1622 0.0549
64 0.5017 0.3856 0.1690
128 1.7005 0.9710 0.5038
256 5.2513 2.6519 2.0804
512 8.0529 6.2976 3.0162Quoted 7 lines
Indeed...
Could you explicit the test you are actually doing ?
Thanks
Quoted 10 lines
It is a dead simple test.
The test starts by forking off X number of tasks
assigning each their own cpu.
Each task then allocs a bit of memory.
All tasks wait on a memory cell for the go order.
We measure the read time starting here.
Once the go order is given they all read a chunk of the selected proc file. I was using/proc/cpuinfoto test.
Once everyone has finished we take the end read time.
Here is the text for those who are curious.
/*------------------------------------------------------------------------------------*/
char *helpstr[] = {
"This test program is a generic template.",
0
};#include <stdio.h>
#include <sys/types.h> #include <unistd.h>
#include <stdlib.h>
#include <sys/wait.h> #include <sys/mman.h>
#include <sched.h>
#include <sys/time.h> #include <string.h>
#include <sys/stat.h> #include <fcntl.h>
#include <linux/unistd.h>
//#include "setup.h"
#define MAXCPUS 4096
#define perrorx(s) do { perror(s); exit(1);} while(0)
#define mb() asm volatile("mfence":::"memory")
#define barrier() asm volatile("": : :"memory")
#define cpu_relax() asm volatile ("rep;nop":::"memory");extern int optind, opterr; extern char *optarg;
static int verbose = 0; static int header = 0; static char *file = "/proc/stat"; static int numtasks = 1; static int repeat = 1; static int bufsize = 1024;
struct control_s {
int ready;
int done;
int go;
int exit;
} *cntl;static cpu_set_t *defmask; static int cpu_set_size;
static void runon_init(void)
{
if (!defmask) {
cpu_set_size = CPU_ALLOC_SIZE(MAXCPUS);
defmask = CPU_ALLOC(MAXCPUS);
if (sched_getaffinity(0, cpu_set_size, defmask) < 0)
perrorx("unexpected failure in runon_init");
}
}static double timeInSeconds(long time_in_microseconds)
{
double temp;
temp = time_in_microseconds;
temp /= 1000000;
return temp;
}static int runon(int cpu)
{
cpu_set_t *mask;
runon_init();
mask = CPU_ALLOC(MAXCPUS);
if (cpu < 0 || cpu >= MAXCPUS)
return -1;
CPU_ZERO_S(cpu_set_size, mask);
CPU_SET_S(cpu, cpu_set_size, mask);
if (sched_setaffinity(0, cpu_set_size, mask) < 0)
return -1;
CPU_FREE(mask);
return 0;
}static long getCurrentTime()
{
struct timeval tp;
long usec;
mb();
gettimeofday(&tp, 0);
usec = tp.tv_sec * 1000000 + tp.tv_usec;
mb();
return usec;
}static void do_help(void)
{
char **p;
for (p = helpstr; *p; p++)
printf("%s\n", *p);
exit(0);
}static void slave(int id)
{
FILE *f;
int i;
char *buf;
runon(id);
buf = malloc(bufsize);
memset(buf, 0, bufsize);
if ((f = fopen(file, "r")) < 0)
perrorx("open failed");
while (fgets(buf, bufsize, f) != NULL) {
}
fclose(f);
(void)__sync_fetch_and_add(&cntl->ready, 1);
while (!cntl->go)
cpu_relax();
for (i = 0; i < repeat; i++) {
if ((f = fopen(file, "r")) < 0)
perrorx("open failed");
while (fgets(buf, bufsize, f) != NULL) {
}
fclose(f);
barrier();
}
(void)__sync_fetch_and_add(&cntl->done, 1);
while (!cntl->exit)
cpu_relax();
exit(0);
}int main(int argc, char **argv)
{
int i, c, stat, er = 0;
static char optstr[] = "b:f:hn:r:v";
unsigned long t, tfork, tready, tread, texit;
opterr = 1;
while ((c = getopt(argc, argv, optstr)) != EOF)
switch (c) {
case 'b':
bufsize = atoi(optarg);
break;
case 'f':
file = optarg;
break;
case 'h':
header++;
break;
case 'n':
numtasks = atoi(optarg);
break;
case 'r':
repeat = atoi(optarg);
break;
case 'v':
verbose++;
break;
case '?':
er = 1;
break;
}
if (er)
do_help();
runon(0);
cntl = mmap(NULL, getpagesize(), PROT_WRITE | PROT_READ, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
tfork = getCurrentTime();
for (i = 0; i < numtasks; i++)
if (fork() == 0)
slave(i + 1);
t = getCurrentTime();
tfork = t - tfork;
tready = t;
while (cntl->ready != numtasks)
usleep(1000);
t = getCurrentTime();
tready = t - tready;
tread = t;
cntl->go = 1;
while (cntl->done != numtasks)
cpu_relax();
t = getCurrentTime();
tread = t - tread;
texit = t;
cntl->exit = 1;
while (wait(&stat) > 0)
usleep(1000);
texit = getCurrentTime() - texit;
if (header) {
printf("File: %s\n", file);
printf("Bufsize: %d\n", bufsize);
printf("Repeats: %d\n", repeat);
printf("%6s%18s%18s%18s%18s%18s\n", "tasks", "fork-sec", "ready-sec", "read-sec", "read/repeat sec",
"texit");
}
printf("%6d%18.6f%18.6f%18.6f%18.6f%18.6f\n", numtasks, timeInSeconds(tfork), timeInSeconds(tready),
timeInSeconds(tread), timeInSeconds(tread) / repeat, timeInSeconds(texit));
return 0;
}Quoted 10 lines · Eric Dumazet
Quoted 6 lines · Eric Dumazet
Thats interesting, but if you really want this to fly, one RCU conversion would be much better ;)
pde_userswould be anatomic_tand you would avoid the spinlock contention.Here is what I had in mind, I would be interested to know how it helps a 512 core machine ;)
Nothing can stop RCU!
After running "modprobe;rmmod" in a loop and "cat" in another loop for a while rmmod got stuck in D-state inside remove_proc_entry() with trace amounts of CPU time being consumed.
It didn't oopsed, though.
Quoted 18 lines
include/linux/proc_fs.h +2
-5@@ -64,16 +64,13 @@ struct proc_dir_entry {* If you're allocating ->proc_fops dynamically, save a pointer * somewhere. */ -const struct file_operations *proc_fops; +const struct file_operations __rcu *proc_fops;struct proc_dir_entry *next, *parent, *subdir;void *data;read_proc_t *read_proc;write_proc_t *write_proc;atomic_t count;/* use count */ -int pde_users;/* number of callers into module in progress */ -struct completion *pde_unload_completion; -struct list_head pde_openers;/* who did ->open, but not ->release */ -spinlock_t pde_unload_lock;/* proc_fops checks and pde_users bumps */ +atomic_t pde_users;/* number of callers into module in progress */
Quoted 7 lines · Alexey Dobriyan
Nothing can stop RCU!
After running "modprobe;rmmod" in a loop and "cat" in another loop for a while rmmod got stuck in D-state inside
remove_proc_entry()with trace amounts of CPU time being consumed.It didn't oopsed, though.
Thanks !
I'll polish this patch once LKS/LPC is over...
What particular module and/or proc file did you use for your tests ?
Quoted 8 lines · Eric Dumazet
Quoted 6 lines · Alexey Dobriyan
Nothing can stop RCU!
After running "modprobe;rmmod" in a loop and "cat" in another loop for a while rmmod got stuck in D-state inside
remove_proc_entry()with trace amounts of CPU time being consumed.
Quoted 7 lines
Quoted 1 line
It didn't oopsed, though.
Thanks !
I'll polish this patch once LKS/LPC is over...
What particular module and/or proc file did you use for your tests ?
Just dummy one.
#include <linux/init.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
static int foo_proc_show(struct seq_file *m, void *v)
{
seq_puts(m, "foo\n");
return 0;
}static int foo_proc_open(struct inode *inode, struct file *file)
{
return single_open(file, foo_proc_show, NULL);
}static const struct file_operations foo_proc_ops = {
.open = foo_proc_open,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};static int __init foo_module_init(void)
{
proc_create("foo", 0, NULL, &foo_proc_ops);
return 0;
}static void __exit foo_module_exit(void)
{
remove_proc_entry("foo", NULL);
}module_init(foo_module_init);module_exit(foo_module_exit);
Quoted 1 line · Eric Dumazet
I'll polish this patch once LKS/LPC is over...
It should oops in the following way (excuse Gmail please):
PDEO is removed from lists
->pde_users is 0
PDE won't be in purge queue -- no ->release while module is alive
Current code removes PDEO and checks if PDE scheduled for removal atomically.
proc_reg_release remove_proc_entry
de->proc_fops = NULL;
release = pde_opener_del(inode, file);
rcu_read_lock();
synchronize_rcu();
fops = rcu_dereference(pde->proc_fops);
if (!fops) {
rcu_read_unlock();
----------------------------------
/* NOP */
while
(atomic_read(&de->pde_users))
...
/* NOP */pde_openers_purge(de, &purge_queue);
/* NOP */
while(!list_empty(&purge_queue))
...
rmmod
if (release)
release(inode, file) /* OOPS */
Quoted 29 lines · Alexey Dobriyan
Quoted 1 line · Eric Dumazet
I'll polish this patch once LKS/LPC is over...
It should oops in the following way (excuse Gmail please):
PDEO is removed from lists
->pde_users is 0
PDE won't be in purge queue -- no ->release while module is aliveCurrent code removes PDEO and checks if PDE scheduled for removal atomically.
proc_reg_releaseremove_proc_entryde->proc_fops = NULL; release = pde_opener_del(inode, file); rcu_read_lock(); synchronize_rcu(); fops = rcu_dereference(pde->proc_fops); if (!fops) { rcu_read_unlock(); ---------------------------------- /* NOP */ while (atomic_read(&de->pde_users)) ... /* NOP */pde_openers_purge(de, &purge_queue); /* NOP */ while
Quoted 6 lines
(!list_empty(&purge_queue))
...
rmmodif (release)
release(inode, file)/* OOPS */
Fix should be trivial, proper module refcount for example.
As I said, I would do that after LKS/LPC, there is no hurry.
Quoted 34 lines · Eric Dumazet
Quoted 33 lines · Alexey Dobriyan
Quoted 1 line · Eric Dumazet
I'll polish this patch once LKS/LPC is over...
It should oops in the following way (excuse Gmail please):
PDEO is removed from lists
->pde_users is 0
PDE won't be in purge queue -- no ->release while module is aliveCurrent code removes PDEO and checks if PDE scheduled for removal atomically.
proc_reg_releaseremove_proc_entryde->proc_fops = NULL; release = pde_opener_del(inode, file); rcu_read_lock(); synchronize_rcu(); fops = rcu_dereference(pde->proc_fops); if (!fops) { rcu_read_unlock(); ---------------------------------- /* NOP */ while (atomic_read(&de->pde_users)) ... /* NOP */pde_openers_purge(de, &purge_queue); /* NOP */ while (!list_empty(&purge_queue)) ... rmmod
Quoted 7 lines
Quoted 2 lines
if (release)
release(inode, file)/* OOPS */Fix should be trivial, proper module refcount for example.
As I said, I would do that after LKS/LPC, there is no hurry.
I should have produced this sooner but there was some higher priority issues. Plus this was the first time I played with the RCU.
Would this fix it?
In proc_reg_release I moved the reference count to proctect the release(inode, file) call.
Also in remove_proc_entry shouldn't we be setting de->proc_fops to NULL with rcu_assign_pointer?
fs/proc/generic.c | 66 ++++------ fs/proc/inode.c | 250 +++++++++++++++++++++++--------------- fs/proc/internal.h | 2 include/linux/proc_fs.h | 7 - 4 files changed, 190 insertions(+), 135 deletions(-)
fs/proc/generic.c +30 -36
@@ -21,7 +21,7 @@#include <linux/namei.h> #include <linux/bitops.h> #include <linux/spinlock.h> -#include <linux/completion.h> +#include <linux/sched.h> #include <asm/uaccess.h> #include "internal.h"
@@ -190,14 +190,16 @@ proc_file_read(struct file *file, char _ {struct proc_dir_entry *pde =PDE (file->f_path.dentry->d_inode);ssize_t rv = -EIO ; +const struct file_operations *fops; -spin_lock (&pde->pde_unload_lock); -if (!pde->proc_fops) { -spin_unlock (&pde->pde_unload_lock); +rcu_read_lock (); + fops =rcu_dereference (pde->proc_fops); +if (!fops) { +rcu_read_unlock ();return rv; } - pde->pde_users++; -spin_unlock (&pde->pde_unload_lock); +atomic_inc (&pde->pde_users); +rcu_read_unlock (); rv =__proc_file_read (file, buf, nbytes, ppos);
@@ -213,13 +215,16 @@ proc_file_write(struct file *file, constssize_t rv = -EIO ;if (pde->write_proc) { -spin_lock (&pde->pde_unload_lock); -if (!pde->proc_fops) { -spin_unlock (&pde->pde_unload_lock); +const struct file_operations *fops; + +rcu_read_lock (); + fops =rcu_dereference (pde->proc_fops); +if (!fops) { +rcu_read_unlock ();return rv; } - pde->pde_users++; -spin_unlock (&pde->pde_unload_lock); +atomic_inc (&pde->pde_users); +rcu_read_unlock ();/* FIXME: does this routine need ppos? probably... */ rv = pde->write_proc (file, buffer, count, pde->data);
@@ -564,7 +569,7 @@ static int proc_register(struct proc_dirif (S_ISDIR (dp->mode)) {if (dp->proc_iops ==NULL ) { - dp->proc_fops = &proc_dir_operations; +RCU_INIT_POINTER (dp->proc_fops, &proc_dir_operations); dp->proc_iops = &proc_dir_inode_operations; } dir->nlink++;
@@ -573,7 +578,7 @@ static int proc_register(struct proc_dir dp->proc_iops = &proc_link_inode_operations; }else if (S_ISREG (dp->mode)) {if (dp->proc_fops ==NULL ) - dp->proc_fops = &proc_file_operations; +RCU_INIT_POINTER (dp->proc_fops, &proc_file_operations);if (dp->proc_iops ==NULL ) dp->proc_iops = &proc_file_inode_operations; }
@@ -625,11 +630,8 @@ static struct proc_dir_entry *__proc_cre ent->mode = mode; ent->nlink = nlink;atomic_set (&ent->count,1 ); - ent->pde_users =0 ; -spin_lock_init (&ent->pde_unload_lock); - ent->pde_unload_completion =NULL ; -INIT_LIST_HEAD (&ent->pde_openers); - out: +atomic_set (&ent->pde_users,0 ); +out:return ent; }
@@ -751,7 +753,7 @@ struct proc_dir_entry *proc_create_data( pde =__proc_create (&parent, name, mode, nlink);if (!pde) goto out; - pde->proc_fops = proc_fops; +rcu_assign_pointer (pde->proc_fops, proc_fops); pde->data = data;if (proc_register (parent, pde) <0 ) goto out_free;
@@ -787,6 +789,7 @@ void remove_proc_entry(const char *name,struct proc_dir_entry *de =NULL ;const char *fn = name;unsigned int len; +LIST_HEAD (purge_queue);spin_lock (&proc_subdir_lock);if (__xlate_proc_name (name, &parent, &fn) !=0 ) {
@@ -809,37 +812,28 @@ void remove_proc_entry(const char *name,return ; } -spin_lock (&de->pde_unload_lock);/* * Stop accepting new callers into module. If you're * dynamically allocating ->proc_fops, save a pointer somewhere. */ de->proc_fops =NULL ; +synchronize_rcu ();/* Wait until all existing callers into module are done. */ -if (de->pde_users >0 ) { -DECLARE_COMPLETION_ONSTACK (c); - -if (!de->pde_unload_completion) - de->pde_unload_completion = &c; - -spin_unlock (&de->pde_unload_lock); - -wait_for_completion (de->pde_unload_completion); - -spin_lock (&de->pde_unload_lock); +while (atomic_read (&de->pde_users)) { +set_current_state (TASK_UNINTERRUPTIBLE ); +schedule (); } + current->state =TASK_RUNNING ; +pde_openers_purge (de, &purge_queue); -while (!list_empty (&de->pde_openers)) { +while (!list_empty (&purge_queue)) {struct pde_opener *pdeo; - pdeo = list_first_entry(&de->pde_openers, struct pde_opener, lh); + pdeo = list_first_entry(&purge_queue, struct pde_opener, lh);list_del (&pdeo->lh); -spin_unlock (&de->pde_unload_lock); pdeo->release (pdeo->inode, pdeo->file);kfree (pdeo); -spin_lock (&de->pde_unload_lock); } -spin_unlock (&de->pde_unload_lock);if (S_ISDIR (de->mode)) parent->nlink--;
fs/proc/inode.c +157 -94
397 lines not on this page · raw message
fs/proc/internal.h +2 -0
@@ -97,12 +97,14 @@ int proc_readdir_de(struct proc_dir_entr filldir_t filldir);struct pde_opener { +struct proc_dir_entry *pde;struct inode *inode;struct file *file;int (*release)(struct inode *,struct file *);struct list_head lh; };void pde_users_dec (struct proc_dir_entry *pde); +void pde_openers_purge (struct proc_dir_entry *pde,struct list_head *queue);extern spinlock_t proc_subdir_lock;
include/linux/proc_fs.h +2 -5
@@ -64,16 +64,13 @@ struct proc_dir_entry {* If you're allocating ->proc_fops dynamically, save a pointer * somewhere. */ -const struct file_operations *proc_fops; +const struct file_operations __rcu *proc_fops;struct proc_dir_entry *next, *parent, *subdir;void *data;read_proc_t *read_proc;write_proc_t *write_proc;atomic_t count;/* use count */ -int pde_users;/* number of callers into module in progress */ -struct completion *pde_unload_completion; -struct list_head pde_openers;/* who did ->open, but not ->release */ -spinlock_t pde_unload_lock;/* proc_fops checks and pde_users bumps */ +atomic_t pde_users;/* number of callers into module in progress */ u8 namelen;char name[]; };