procfs: Improve Scaling in proc
Subsystems (2)
PATCH v2 1/1 procfs: Improve Scaling in proc
I am currently tracking a hotlock reported by a customer on a large, 512 cores, system, I am currently running 3.7.0 rc1 but the issue looks like it has been this way for a very long time.
The offending lock is proc_dir_entry->pde_unload_lock.
This patch converts the replaces the lock with the rcu. It is a refresh of what was orignally suggested by Eric Dumazet. I refreshed it to the 3.7.
Supporting numbers, lower is better, they are from the test I posted earlier.
cpuinfo baseline Rcu tasks read-sec read-sec 1 0.0141 0.0141 2 0.0140 0.0142 4 0.0140 0.0141 8 0.0145 0.0140 16 0.0553 0.0168 32 0.1688 0.0549 64 0.5017 0.1690 128 1.7005 0.5038 256 5.2513 2.0804 512 8.0529 3.0162
V2 Use the completion mechanism instead of a polling loop.
--- fs/proc/generic.c | 56 +++++----- fs/proc/inode.c | 252 +++++++++++++++++++++++++++++------------------ fs/proc/internal.h | 2 + include/linux/proc_fs.h | 6 +- 4 files changed, 190 insertions(+), 126 deletions(-)
fs/proc/generic.c +28 -28
@@ -22,6 +22,7 @@#include <linux/bitops.h> #include <linux/spinlock.h> #include <linux/completion.h> +#include <linux/sched.h> #include <asm/uaccess.h> #include "internal.h"
@@ -190,14 +191,16 @@ proc_file_read(struct file *file, char _ {struct proc_dir_entry *pde =PDE (file->f_path.dentry->d_inode);ssize_t rv = -EIO ; +const struct file_operations *fops; -spin_lock (&pde->pde_unload_lock); -if (!pde->proc_fops) { -spin_unlock (&pde->pde_unload_lock); +rcu_read_lock (); + fops =rcu_dereference (pde->proc_fops); +if (!fops) { +rcu_read_unlock ();return rv; } - pde->pde_users++; -spin_unlock (&pde->pde_unload_lock); +atomic_inc (&pde->pde_users); +rcu_read_unlock (); rv =__proc_file_read (file, buf, nbytes, ppos);
@@ -213,13 +216,16 @@ proc_file_write(struct file *file, constssize_t rv = -EIO ;if (pde->write_proc) { -spin_lock (&pde->pde_unload_lock); -if (!pde->proc_fops) { -spin_unlock (&pde->pde_unload_lock); +const struct file_operations *fops; + +rcu_read_lock (); + fops =rcu_dereference (pde->proc_fops); +if (!fops) { +rcu_read_unlock ();return rv; } - pde->pde_users++; -spin_unlock (&pde->pde_unload_lock); +atomic_inc (&pde->pde_users); +rcu_read_unlock ();/* FIXME: does this routine need ppos? probably... */ rv = pde->write_proc (file, buffer, count, pde->data);
@@ -564,7 +570,7 @@ static int proc_register(struct proc_dirif (S_ISDIR (dp->mode)) {if (dp->proc_iops ==NULL ) { - dp->proc_fops = &proc_dir_operations; +RCU_INIT_POINTER (dp->proc_fops, &proc_dir_operations); dp->proc_iops = &proc_dir_inode_operations; } dir->nlink++;
@@ -573,7 +579,7 @@ static int proc_register(struct proc_dir dp->proc_iops = &proc_link_inode_operations; }else if (S_ISREG (dp->mode)) {if (dp->proc_fops ==NULL ) - dp->proc_fops = &proc_file_operations; +RCU_INIT_POINTER (dp->proc_fops, &proc_file_operations);if (dp->proc_iops ==NULL ) dp->proc_iops = &proc_file_inode_operations; }
@@ -626,8 +632,7 @@ static struct proc_dir_entry *__proc_cre ent->mode = mode; ent->nlink = nlink;atomic_set (&ent->count,1 ); -spin_lock_init (&ent->pde_unload_lock); -INIT_LIST_HEAD (&ent->pde_openers); +atomic_set (&ent->pde_users,1 ); out:return ent; }
@@ -750,7 +755,7 @@ struct proc_dir_entry *proc_create_data( pde =__proc_create (&parent, name, mode, nlink);if (!pde) goto out; - pde->proc_fops = proc_fops; +rcu_assign_pointer (pde->proc_fops, proc_fops); pde->data = data;if (proc_register (parent, pde) <0 ) goto out_free;
@@ -786,6 +791,7 @@ void remove_proc_entry(const char *name,struct proc_dir_entry *de =NULL ;const char *fn = name;unsigned int len; +LIST_HEAD (purge_queue);spin_lock (&proc_subdir_lock);if (__xlate_proc_name (name, &parent, &fn) !=0 ) {
@@ -808,37 +814,31 @@ void remove_proc_entry(const char *name,return ; } -spin_lock (&de->pde_unload_lock);/* * Stop accepting new callers into module. If you're * dynamically allocating ->proc_fops, save a pointer somewhere. */ - de->proc_fops =NULL ; -/* Wait until all existing callers into module are done. */ -if (de->pde_users >0 ) { -DECLARE_COMPLETION_ONSTACK (c); +rcu_assign_pointer (de->proc_fops,NULL ); +synchronize_rcu (); +DECLARE_COMPLETION_ONSTACK (c); +if (!atomic_dec_and_test (&de->pde_users)) {if (!de->pde_unload_completion) de->pde_unload_completion = &c; -spin_unlock (&de->pde_unload_lock); -wait_for_completion (de->pde_unload_completion); -spin_lock (&de->pde_unload_lock); } +pde_openers_purge (de, &purge_queue); -while (!list_empty (&de->pde_openers)) { +while (!list_empty (&purge_queue)) {struct pde_opener *pdeo; - pdeo = list_first_entry(&de->pde_openers, struct pde_opener, lh); + pdeo = list_first_entry(&purge_queue, struct pde_opener, lh);list_del (&pdeo->lh); -spin_unlock (&de->pde_unload_lock); pdeo->release (pdeo->inode, pdeo->file);kfree (pdeo); -spin_lock (&de->pde_unload_lock); } -spin_unlock (&de->pde_unload_lock);if (S_ISDIR (de->mode)) parent->nlink--;
fs/proc/inode.c +157 -94
397 lines not on this page · raw message
fs/proc/internal.h +2 -0
@@ -145,12 +145,14 @@ int proc_readdir_de(struct proc_dir_entr filldir_t filldir);struct pde_opener { +struct proc_dir_entry *pde;struct inode *inode;struct file *file;int (*release)(struct inode *,struct file *);struct list_head lh; };void pde_users_dec (struct proc_dir_entry *pde); +void pde_openers_purge (struct proc_dir_entry *pde,struct list_head *queue);extern spinlock_t proc_subdir_lock;
include/linux/proc_fs.h +2 -4
@@ -64,16 +64,14 @@ struct proc_dir_entry {* If you're allocating ->proc_fops dynamically, save a pointer * somewhere. */ -const struct file_operations *proc_fops; +const struct file_operations __rcu *proc_fops;struct proc_dir_entry *next, *parent, *subdir;void *data;read_proc_t *read_proc;write_proc_t *write_proc;atomic_t count;/* use count */ -int pde_users;/* number of callers into module in progress */ struct completion *pde_unload_completion; -struct list_head pde_openers;/* who did ->open, but not ->release */ -spinlock_t pde_unload_lock;/* proc_fops checks and pde_users bumps */ +atomic_t pde_users;/* number of callers into module in progress */ u8 namelen;char name[]; };
PATCH v3 1/1 procfs: Improve Scaling in proc
I am currently tracking a hotlock reported by a customer on a large system, 512 cores. I am currently running 3.7.1 but the issue looks like it has been this way for a very long time.
The offending lock is proc_dir_entry->pde_unload_lock.
This patch converts the replaces the lock with the rcu. However the pde_openers list still is controlled by a spin lock. I tested on a 4096 machine and the lock doesn't seem hot at least according to perf.
This is a refresh of what was orignally suggested by Eric Dumazet some time ago.
Supporting numbers, lower is better, they are from the test I posted earlier.
cpuinfo baseline Rcu tasks read-sec read-sec 1 0.0141 0.0141 2 0.0140 0.0142 4 0.0140 0.0141 8 0.0145 0.0140 16 0.0553 0.0168 32 0.1688 0.0549 64 0.5017 0.1690 128 1.7005 0.5038 256 5.2513 2.0804 512 8.0529 3.0162
--- fs/proc/generic.c | 58 +++++++++--------- fs/proc/inode.c | 156 ++++++++++++++++++++++++++--------------------- include/linux/proc_fs.h | 6 +- 3 files changed, 118 insertions(+), 102 deletions(-)
fs/proc/generic.c +29 -29
@@ -191,13 +191,16 @@ proc_file_read(struct file *file, char __user *buf, size_t nbytes,struct proc_dir_entry *pde =PDE (file->f_path.dentry->d_inode);ssize_t rv = -EIO ; -spin_lock (&pde->pde_unload_lock); -if (!pde->proc_fops) { -spin_unlock (&pde->pde_unload_lock); +const struct file_operations *fops; + +rcu_read_lock (); + fops =rcu_dereference (pde->proc_fops); +if (!fops) { +rcu_read_unlock ();return rv; } - pde->pde_users++; -spin_unlock (&pde->pde_unload_lock); +atomic_inc (&pde->pde_users); +rcu_read_unlock (); rv =__proc_file_read (file, buf, nbytes, ppos);
@@ -213,13 +216,16 @@ proc_file_write(struct file *file, const char __user *buffer,ssize_t rv = -EIO ;if (pde->write_proc) { -spin_lock (&pde->pde_unload_lock); -if (!pde->proc_fops) { -spin_unlock (&pde->pde_unload_lock); +const struct file_operations *fops; + +rcu_read_lock (); + fops =rcu_dereference (pde->proc_fops); +if (!fops) { +rcu_read_unlock ();return rv; } - pde->pde_users++; -spin_unlock (&pde->pde_unload_lock); +atomic_inc (&pde->pde_users); +rcu_read_unlock ();/* FIXME: does this routine need ppos? probably... */ rv = pde->write_proc (file, buffer, count, pde->data);
@@ -564,7 +570,7 @@ static int proc_register(struct proc_dir_entry * dir, struct proc_dir_entry * dpif (S_ISDIR (dp->mode)) {if (dp->proc_iops ==NULL ) { - dp->proc_fops = &proc_dir_operations; +RCU_INIT_POINTER (dp->proc_fops, &proc_dir_operations); dp->proc_iops = &proc_dir_inode_operations; } dir->nlink++;
@@ -573,7 +579,7 @@ static int proc_register(struct proc_dir_entry * dir, struct proc_dir_entry * dp dp->proc_iops = &proc_link_inode_operations; }else if (S_ISREG (dp->mode)) {if (dp->proc_fops ==NULL ) - dp->proc_fops = &proc_file_operations; +RCU_INIT_POINTER (dp->proc_fops, &proc_file_operations);if (dp->proc_iops ==NULL ) dp->proc_iops = &proc_file_inode_operations; }
@@ -626,7 +632,8 @@ static struct proc_dir_entry *__proc_create(struct proc_dir_entry **parent, ent->mode = mode; ent->nlink = nlink;atomic_set (&ent->count,1 ); -spin_lock_init (&ent->pde_unload_lock); +atomic_set (&ent->pde_users,1 ); +spin_lock_init (&ent->pde_openers_lock);INIT_LIST_HEAD (&ent->pde_openers); out:return ent;
@@ -750,7 +757,7 @@ struct proc_dir_entry *proc_create_data(const char *name, umode_t mode, pde =__proc_create (&parent, name, mode, nlink);if (!pde) goto out; - pde->proc_fops = proc_fops; +rcu_assign_pointer (pde->proc_fops, proc_fops); pde->data = data;if (proc_register (parent, pde) <0 ) goto out_free;
@@ -808,37 +815,30 @@ void remove_proc_entry(const char *name, struct proc_dir_entry *parent)return ; } -spin_lock (&de->pde_unload_lock);/* * Stop accepting new callers into module. If you're * dynamically allocating ->proc_fops, save a pointer somewhere. */ - de->proc_fops =NULL ; -/* Wait until all existing callers into module are done. */ -if (de->pde_users >0 ) { -DECLARE_COMPLETION_ONSTACK (c); - -if (!de->pde_unload_completion) - de->pde_unload_completion = &c; -spin_unlock (&de->pde_unload_lock); +rcu_assign_pointer (de->proc_fops,NULL ); +synchronize_rcu (); +/* Wait until all existing callers into module are done. */ +DECLARE_COMPLETION_ONSTACK (c); + de->pde_unload_completion = &c; +if (!atomic_dec_and_test (&de->pde_users))wait_for_completion (de->pde_unload_completion); -spin_lock (&de->pde_unload_lock); - } - +spin_lock (&de->pde_openers_lock);while (!list_empty (&de->pde_openers)) {struct pde_opener *pdeo; pdeo = list_first_entry(&de->pde_openers, struct pde_opener, lh);list_del (&pdeo->lh); -spin_unlock (&de->pde_unload_lock); pdeo->release (pdeo->inode, pdeo->file);kfree (pdeo); -spin_lock (&de->pde_unload_lock); } -spin_unlock (&de->pde_unload_lock); +spin_unlock (&de->pde_openers_lock);if (S_ISDIR (de->mode)) parent->nlink--;
fs/proc/inode.c +86 -70
301 lines not on this page · raw message
include/linux/proc_fs.h +3 -3
@@ -64,16 +64,16 @@ struct proc_dir_entry {* If you're allocating ->proc_fops dynamically, save a pointer * somewhere. */ -const struct file_operations *proc_fops; +const struct file_operations __rcu *proc_fops;struct proc_dir_entry *next, *parent, *subdir;void *data;read_proc_t *read_proc;write_proc_t *write_proc;atomic_t count;/* use count */ -int pde_users;/* number of callers into module in progress */ +atomic_t pde_users;/* number of callers into module in progress */ struct completion *pde_unload_completion; +spinlock_t pde_openers_lock;struct list_head pde_openers;/* who did ->open, but not ->release */ -spinlock_t pde_unload_lock;/* proc_fops checks and pde_users bumps */ u8 namelen;char name[]; };
Signature · 1 line
1.6.0.2