procfs: Improve Scaling in proc
Subsystems (3)
PATCH v4 1/1 procfs: Improve Scaling in proc
I am currently tracking a hotlock reported by a customer on a large system, 512 cores. I am currently running 3.8-rc7 but the issue looks like it has been this way for a very long time.
The offending lock is proc_dir_entry->pde_unload_lock.
This patch converts the lock to use the rcu. However the pde_openers list still is controlled by a spin lock. I tested on a 4096 machine and the lock doesn't seem hot at least according to perf.
This is a refresh of what was orignally suggested by Eric Dumazet some time ago. I have also taken in some comments from Andrew and several other people whose names escape me but I am quite grateful too.
Supporting numbers, lower is better, they are from the test I posted earlier.
cpuinfo baseline Rcu tasks read-sec read-sec 1 0.0141 0.0141 2 0.0140 0.0142 4 0.0140 0.0141 8 0.0145 0.0140 16 0.0553 0.0168 32 0.1688 0.0549 64 0.5017 0.1690 128 1.7005 0.5038 256 5.2513 2.0804 512 8.0529 3.0162
--- fs/proc/generic.c | 62 +++++++++---------- fs/proc/inode.c | 161 ++++++++++++++++++++++++++---------------------- include/linux/proc_fs.h | 6 +- 3 files changed, 123 insertions(+), 106 deletions(-)
fs/proc/generic.c +31 -31
@@ -190,14 +190,16 @@ proc_file_read(struct file *file, char __user *buf, size_t nbytes, {struct proc_dir_entry *pde =PDE (file->f_path.dentry->d_inode);ssize_t rv = -EIO ; +const struct file_operations *fops; -spin_lock (&pde->pde_unload_lock); -if (!pde->proc_fops) { -spin_unlock (&pde->pde_unload_lock); +rcu_read_lock (); + fops =rcu_dereference (pde->proc_fops); +if (!fops) { +rcu_read_unlock ();return rv; } - pde->pde_users++; -spin_unlock (&pde->pde_unload_lock); +atomic_inc (&pde->pde_users); +rcu_read_unlock (); rv =__proc_file_read (file, buf, nbytes, ppos);
@@ -213,13 +215,16 @@ proc_file_write(struct file *file, const char __user *buffer,ssize_t rv = -EIO ;if (pde->write_proc) { -spin_lock (&pde->pde_unload_lock); -if (!pde->proc_fops) { -spin_unlock (&pde->pde_unload_lock); +const struct file_operations *fops; + +rcu_read_lock (); + fops =rcu_dereference (pde->proc_fops); +if (!fops) { +rcu_read_unlock ();return rv; } - pde->pde_users++; -spin_unlock (&pde->pde_unload_lock); +atomic_inc (&pde->pde_users); +rcu_read_unlock ();/* FIXME: does this routine need ppos? probably... */ rv = pde->write_proc (file, buffer, count, pde->data);
@@ -558,7 +563,7 @@ static int proc_register(struct proc_dir_entry * dir, struct proc_dir_entry * dpif (S_ISDIR (dp->mode)) {if (dp->proc_iops ==NULL ) { - dp->proc_fops = &proc_dir_operations; +RCU_INIT_POINTER (dp->proc_fops, &proc_dir_operations); dp->proc_iops = &proc_dir_inode_operations; } dir->nlink++;
@@ -567,7 +572,7 @@ static int proc_register(struct proc_dir_entry * dir, struct proc_dir_entry * dp dp->proc_iops = &proc_link_inode_operations; }else if (S_ISREG (dp->mode)) {if (dp->proc_fops ==NULL ) - dp->proc_fops = &proc_file_operations; +RCU_INIT_POINTER (dp->proc_fops, &proc_file_operations);if (dp->proc_iops ==NULL ) dp->proc_iops = &proc_file_inode_operations; }
@@ -620,7 +625,8 @@ static struct proc_dir_entry *__proc_create(struct proc_dir_entry **parent, ent->mode = mode; ent->nlink = nlink;atomic_set (&ent->count,1 ); -spin_lock_init (&ent->pde_unload_lock); +atomic_set (&ent->pde_users,1 ); +spin_lock_init (&ent->pde_openers_lock);INIT_LIST_HEAD (&ent->pde_openers); out:return ent;
@@ -744,7 +750,7 @@ struct proc_dir_entry *proc_create_data(const char *name, umode_t mode, pde =__proc_create (&parent, name, mode, nlink);if (!pde) goto out; - pde->proc_fops = proc_fops; +rcu_assign_pointer (pde->proc_fops, proc_fops); pde->data = data;if (proc_register (parent, pde) <0 ) goto out_free;
@@ -780,6 +786,7 @@ void remove_proc_entry(const char *name, struct proc_dir_entry *parent)struct proc_dir_entry *de =NULL ;const char *fn = name;unsigned int len; +DECLARE_COMPLETION_ONSTACK (c);spin_lock (&proc_subdir_lock);if (__xlate_proc_name (name, &parent, &fn) !=0 ) {
@@ -802,37 +809,30 @@ void remove_proc_entry(const char *name, struct proc_dir_entry *parent)return ; } -spin_lock (&de->pde_unload_lock);/* * Stop accepting new callers into module. If you're * dynamically allocating ->proc_fops, save a pointer somewhere. */ - de->proc_fops =NULL ; -/* Wait until all existing callers into module are done. */ -if (de->pde_users >0 ) { -DECLARE_COMPLETION_ONSTACK (c); - -if (!de->pde_unload_completion) - de->pde_unload_completion = &c; - -spin_unlock (&de->pde_unload_lock); - +rcu_assign_pointer (de->proc_fops,NULL ); +synchronize_rcu (); +/* +* Wait until all existing callers into module are done. +* Once pde_users hits zero we are free to clean out pde_openers. +*/ + de->pde_unload_completion = &c; +if (!atomic_dec_and_test (&de->pde_users))wait_for_completion (de->pde_unload_completion); -spin_lock (&de->pde_unload_lock); - } - +spin_lock (&de->pde_openers_lock);while (!list_empty (&de->pde_openers)) {struct pde_opener *pdeo; pdeo = list_first_entry(&de->pde_openers, struct pde_opener, lh);list_del (&pdeo->lh); -spin_unlock (&de->pde_unload_lock); pdeo->release (pdeo->inode, pdeo->file);kfree (pdeo); -spin_lock (&de->pde_unload_lock); } -spin_unlock (&de->pde_unload_lock); +spin_unlock (&de->pde_openers_lock);if (S_ISDIR (de->mode)) parent->nlink--;
fs/proc/inode.c +89 -72
304 lines not on this page · raw message
include/linux/proc_fs.h +3 -3
@@ -68,16 +68,16 @@ struct proc_dir_entry {* If you're allocating ->proc_fops dynamically, save a pointer * somewhere. */ -const struct file_operations *proc_fops; +const struct file_operations __rcu *proc_fops;struct proc_dir_entry *next, *parent, *subdir;void *data;read_proc_t *read_proc;write_proc_t *write_proc;atomic_t count;/* use count */ -int pde_users;/* number of callers into module in progress */ +atomic_t pde_users;/* number of callers into module in progress */ struct completion *pde_unload_completion; +spinlock_t pde_openers_lock;struct list_head pde_openers;/* who did ->open, but not ->release */ -spinlock_t pde_unload_lock;/* proc_fops checks and pde_users bumps */ u8 namelen;char name[]; };
Signature · 1 line
1.8.1.2
PATCH v5 1/1 procfs: Improve Scaling in proc
I am currently tracking a hotlock reported by a customer on a large system, 512 cores. I am currently running 3.8-rc7 but the issue looks like it has been this way for a very long time.
The offending lock is proc_dir_entry->pde_unload_lock.
This patch converts the lock to use the rcu. However the pde_openers list still is controlled by a spin lock. I tested on a 4096 machine and the lock doesn't seem hot at least according to perf.
This is a refresh of what was orignally suggested by Eric Dumazet some time ago. I have also taken in some comments from Andrew and several other people whose names escape me but I am quite grateful too.
Supporting numbers, lower is better, they are from the test I posted earlier.
cpuinfo baseline Rcu tasks read-sec read-sec 1 0.0141 0.0141 2 0.0140 0.0142 4 0.0140 0.0141 8 0.0145 0.0140 16 0.0553 0.0168 32 0.1688 0.0549 64 0.5017 0.1690 128 1.7005 0.5038 256 5.2513 2.0804 512 8.0529 3.0162
v4: Corrected some comments and warnings.
v5: Corrected some warnings from sparce including the supplied by Sasha.
--- fs/proc/generic.c | 62 ++++++++--------- fs/proc/inode.c | 176 +++++++++++++++++++++++++++--------------------- include/linux/proc_fs.h | 6 +- 3 files changed, 133 insertions(+), 111 deletions(-)
fs/proc/generic.c +31 -31
@@ -190,14 +190,16 @@ proc_file_read(struct file *file, char __user *buf, size_t nbytes, {struct proc_dir_entry *pde =PDE (file->f_path.dentry->d_inode);ssize_t rv = -EIO ; +const struct file_operations *fops; -spin_lock (&pde->pde_unload_lock); -if (!pde->proc_fops) { -spin_unlock (&pde->pde_unload_lock); +rcu_read_lock (); + fops =rcu_dereference (pde->proc_fops); +if (!fops) { +rcu_read_unlock ();return rv; } - pde->pde_users++; -spin_unlock (&pde->pde_unload_lock); +atomic_inc (&pde->pde_users); +rcu_read_unlock (); rv =__proc_file_read (file, buf, nbytes, ppos);
@@ -213,13 +215,16 @@ proc_file_write(struct file *file, const char __user *buffer,ssize_t rv = -EIO ;if (pde->write_proc) { -spin_lock (&pde->pde_unload_lock); -if (!pde->proc_fops) { -spin_unlock (&pde->pde_unload_lock); +const struct file_operations *fops; + +rcu_read_lock (); + fops =rcu_dereference (pde->proc_fops); +if (!fops) { +rcu_read_unlock ();return rv; } - pde->pde_users++; -spin_unlock (&pde->pde_unload_lock); +atomic_inc (&pde->pde_users); +rcu_read_unlock ();/* FIXME: does this routine need ppos? probably... */ rv = pde->write_proc (file, buffer, count, pde->data);
@@ -558,7 +563,7 @@ static int proc_register(struct proc_dir_entry * dir, struct proc_dir_entry * dpif (S_ISDIR (dp->mode)) {if (dp->proc_iops ==NULL ) { - dp->proc_fops = &proc_dir_operations; +RCU_INIT_POINTER (dp->proc_fops, &proc_dir_operations); dp->proc_iops = &proc_dir_inode_operations; } dir->nlink++;
@@ -567,7 +572,7 @@ static int proc_register(struct proc_dir_entry * dir, struct proc_dir_entry * dp dp->proc_iops = &proc_link_inode_operations; }else if (S_ISREG (dp->mode)) {if (dp->proc_fops ==NULL ) - dp->proc_fops = &proc_file_operations; +RCU_INIT_POINTER (dp->proc_fops, &proc_file_operations);if (dp->proc_iops ==NULL ) dp->proc_iops = &proc_file_inode_operations; }
@@ -620,7 +625,8 @@ static struct proc_dir_entry *__proc_create(struct proc_dir_entry **parent, ent->mode = mode; ent->nlink = nlink;atomic_set (&ent->count,1 ); -spin_lock_init (&ent->pde_unload_lock); +atomic_set (&ent->pde_users,1 ); +spin_lock_init (&ent->pde_openers_lock);INIT_LIST_HEAD (&ent->pde_openers); out:return ent;
@@ -744,7 +750,7 @@ struct proc_dir_entry *proc_create_data(const char *name, umode_t mode, pde =__proc_create (&parent, name, mode, nlink);if (!pde) goto out; - pde->proc_fops = proc_fops; +rcu_assign_pointer (pde->proc_fops, proc_fops); pde->data = data;if (proc_register (parent, pde) <0 ) goto out_free;
@@ -780,6 +786,7 @@ void remove_proc_entry(const char *name, struct proc_dir_entry *parent)struct proc_dir_entry *de =NULL ;const char *fn = name;unsigned int len; +DECLARE_COMPLETION_ONSTACK (c);spin_lock (&proc_subdir_lock);if (__xlate_proc_name (name, &parent, &fn) !=0 ) {
@@ -802,37 +809,30 @@ void remove_proc_entry(const char *name, struct proc_dir_entry *parent)return ; } -spin_lock (&de->pde_unload_lock);/* * Stop accepting new callers into module. If you're * dynamically allocating ->proc_fops, save a pointer somewhere. */ - de->proc_fops =NULL ; -/* Wait until all existing callers into module are done. */ -if (de->pde_users >0 ) { -DECLARE_COMPLETION_ONSTACK (c); - -if (!de->pde_unload_completion) - de->pde_unload_completion = &c; - -spin_unlock (&de->pde_unload_lock); - +rcu_assign_pointer (de->proc_fops,NULL ); +synchronize_rcu (); +/* +* Wait until all existing callers into module are done. +* Once pde_users hits zero we are free to clean out pde_openers. +*/ + de->pde_unload_completion = &c; +if (!atomic_dec_and_test (&de->pde_users))wait_for_completion (de->pde_unload_completion); -spin_lock (&de->pde_unload_lock); - } - +spin_lock (&de->pde_openers_lock);while (!list_empty (&de->pde_openers)) {struct pde_opener *pdeo; pdeo = list_first_entry(&de->pde_openers, struct pde_opener, lh);list_del (&pdeo->lh); -spin_unlock (&de->pde_unload_lock); pdeo->release (pdeo->inode, pdeo->file);kfree (pdeo); -spin_lock (&de->pde_unload_lock); } -spin_unlock (&de->pde_unload_lock); +spin_unlock (&de->pde_openers_lock);if (S_ISDIR (de->mode)) parent->nlink--;
fs/proc/inode.c +99 -77
339 lines not on this page · raw message
include/linux/proc_fs.h +3 -3
@@ -68,16 +68,16 @@ struct proc_dir_entry {* If you're allocating ->proc_fops dynamically, save a pointer * somewhere. */ -const struct file_operations *proc_fops; +const struct file_operations __rcu *proc_fops;struct proc_dir_entry *next, *parent, *subdir;void *data;read_proc_t *read_proc;write_proc_t *write_proc;atomic_t count;/* use count */ -int pde_users;/* number of callers into module in progress */ +atomic_t pde_users;/* number of callers into module in progress */ struct completion *pde_unload_completion; +spinlock_t pde_openers_lock;struct list_head pde_openers;/* who did ->open, but not ->release */ -spinlock_t pde_unload_lock;/* proc_fops checks and pde_users bumps */ u8 namelen;char name[]; };
Signature · 1 line
1.8.1.2
PATCH 1/1 dgrp: staging: cleanup sparse warnings
In reply to Nathan ZimmerA cleanup patch to remove sparse warnings caused by my other patch "procfs: Improve Scaling in proc" since now proc_fops is protected by the rcu.
--- drivers/staging/dgrp/dgrp_dpa_ops.c | 2 +- drivers/staging/dgrp/dgrp_mon_ops.c | 2 +- drivers/staging/dgrp/dgrp_net_ops.c | 2 +- drivers/staging/dgrp/dgrp_ports_ops.c | 2 +- drivers/staging/dgrp/dgrp_specproc.c | 6 ++++-- 5 files changed, 8 insertions(+), 6 deletions(-)
drivers/staging/dgrp/dgrp_dpa_ops.c +1 -1
@@ -116,7 +116,7 @@ void dgrp_register_dpa_hook(struct proc_dir_entry *de)struct nd_struct *node = de->data; de->proc_iops = &dpa_inode_ops; - de->proc_fops = &dpa_ops; +rcu_assign_pointer (de->proc_fops, &dpa_ops); node->nd_dpa_de = de;spin_lock_init (&node->nd_dpa_lock);
drivers/staging/dgrp/dgrp_mon_ops.c +1 -1
@@ -66,7 +66,7 @@ void dgrp_register_mon_hook(struct proc_dir_entry *de)struct nd_struct *node = de->data; de->proc_iops = &mon_inode_ops; - de->proc_fops = &mon_ops; +rcu_assign_pointer (de->proc_fops, &mon_ops); node->nd_mon_de = de;sema_init (&node->nd_mon_semaphore,1 ); }
drivers/staging/dgrp/dgrp_net_ops.c +1 -1
@@ -90,7 +90,7 @@ void dgrp_register_net_hook(struct proc_dir_entry *de)struct nd_struct *node = de->data; de->proc_iops = &net_inode_ops; - de->proc_fops = &net_ops; +rcu_assign_pointer (de->proc_fops, &net_ops); node->nd_net_de = de;sema_init (&node->nd_net_semaphore,1 ); node->nd_state =NS_CLOSED ;
drivers/staging/dgrp/dgrp_ports_ops.c +1 -1
@@ -65,7 +65,7 @@ void dgrp_register_ports_hook(struct proc_dir_entry *de)struct nd_struct *node = de->data; de->proc_iops = &ports_inode_ops; - de->proc_fops = &ports_ops; +rcu_assign_pointer (de->proc_fops, &ports_ops); node->nd_ports_de = de; }
drivers/staging/dgrp/dgrp_specproc.c +4 -2
@@ -268,9 +268,11 @@ static void register_proc_table(struct dgrp_proc_entry *table,if (!table->child) { de->proc_iops = &proc_inode_ops;if (table->proc_file_ops) - de->proc_fops = table->proc_file_ops; +rcu_assign_pointer (de->proc_fops, + table->proc_file_ops);else - de->proc_fops = &dgrp_proc_file_ops; +rcu_assign_pointer (de->proc_fops, + &dgrp_proc_file_ops); } } table->de = de;
Signature · 1 line
1.8.1.2